openvidu-browser: MVC Virtual Background

openvidu-browser/src/OpenVidu/OpenVidu.ts

@@ -133,6 +133,10 @@ export class OpenVidu {
    * @hidden
    */
   isPro: boolean = false;
+  /**
+   * @hidden
+   */
+  isEnterprise: boolean = false;
   /**
    * @hidden
    */

openvidu-browser/src/OpenVidu/Publisher.ts

@@ -332,28 +332,6 @@ export class Publisher extends StreamManager {
      * @returns A Promise (to which you can optionally subscribe to) that is resolved if the track was successfully replaced and rejected with an Error object in other case
      */
     async replaceTrack(track: MediaStreamTrack): Promise<void> {
-
-        const replaceTrackInMediaStream = (): Promise<void> => {
-            return new Promise((resolve, reject) => {
-                const mediaStream: MediaStream = this.stream.displayMyRemote() ? this.stream.localMediaStreamWhenSubscribedToRemote! : this.stream.getMediaStream();
-                let removedTrack: MediaStreamTrack;
-                if (track.kind === 'video') {
-                    removedTrack = mediaStream.getVideoTracks()[0];
-                    this.stream.lastVideoTrackConstraints = track.getConstraints();
-                } else {
-                    removedTrack = mediaStream.getAudioTracks()[0];
-                }
-                mediaStream.removeTrack(removedTrack);
-                removedTrack.stop();
-                mediaStream.addTrack(track);
-                if (track.kind === 'video' && this.stream.isLocalStreamPublished) {
-                    this.openvidu.sendNewVideoDimensionsIfRequired(this, 'trackReplaced', 50, 30);
-                    this.session.sendVideoData(this.stream.streamManager, 5, true, 5);
-                }
-                return resolve();
-            });
-        }
-
         // Set field "enabled" of the new track to the previous value
         const trackOriginalEnabledValue: boolean = track.enabled;
         if (track.kind === 'video') {
@@ -366,10 +344,10 @@ export class Publisher extends StreamManager {
                 // Only if the Publisher has been published is necessary to call native Web API RTCRtpSender.replaceTrack
                 // If it has not been published yet, replacing it on the MediaStream object is enough
                 await this.replaceTrackInRtcRtpSender(track);
-                return await replaceTrackInMediaStream();
+                return await this.replaceTrackInMediaStream(track);
             } else {
                 // Publisher not published. Simply replace the track on the local MediaStream
-                return await replaceTrackInMediaStream();
+                return await this.replaceTrackInMediaStream(track);
             }
         } catch (error) {
             track.enabled = trackOriginalEnabledValue;
@@ -751,6 +729,26 @@ export class Publisher extends StreamManager {
         this.videoReference.srcObject = mediaStream;
     }
 
+    /**
+     * @hidden
+     */
+    async replaceTrackInMediaStream(track: MediaStreamTrack): Promise<void> {
+        const mediaStream: MediaStream = this.stream.displayMyRemote() ? this.stream.localMediaStreamWhenSubscribedToRemote! : this.stream.getMediaStream();
+        let removedTrack: MediaStreamTrack;
+        if (track.kind === 'video') {
+            removedTrack = mediaStream.getVideoTracks()[0];
+            this.stream.lastVideoTrackConstraints = track.getConstraints();
+        } else {
+            removedTrack = mediaStream.getAudioTracks()[0];
+        }
+        mediaStream.removeTrack(removedTrack);
+        removedTrack.stop();
+        mediaStream.addTrack(track);
+        if (track.kind === 'video' && this.stream.isLocalStreamPublished) {
+            this.openvidu.sendNewVideoDimensionsIfRequired(this, 'trackReplaced', 50, 30);
+            this.session.sendVideoData(this.stream.streamManager, 5, true, 5);
+        }
+    }
 
     /* Private methods */
 
@@ -768,27 +766,23 @@ export class Publisher extends StreamManager {
         }
     }
 
-    private replaceTrackInRtcRtpSender(track: MediaStreamTrack): Promise<void> {
+    private async replaceTrackInRtcRtpSender(track: MediaStreamTrack): Promise<void> {
-        return new Promise((resolve, reject) => {
         const senders: RTCRtpSender[] = this.stream.getRTCPeerConnection().getSenders();
         let sender: RTCRtpSender | undefined;
         if (track.kind === 'video') {
             sender = senders.find(s => !!s.track && s.track.kind === 'video');
             if (!sender) {
-                    return reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object'));
+                throw new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object');
             }
         } else if (track.kind === 'audio') {
             sender = senders.find(s => !!s.track && s.track.kind === 'audio');
             if (!sender) {
-                    return reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object'));
+                throw new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object');
             }
         } else {
-                return reject(new Error('Unknown track kind ' + track.kind));
+            throw new Error('Unknown track kind ' + track.kind);
         }
-            (sender as RTCRtpSender).replaceTrack(track)
+        await (sender as RTCRtpSender).replaceTrack(track);
-                .then(() => resolve())
-                .catch(error => reject(error));
-        });
     }
 
 }

openvidu-browser/src/OpenVidu/Session.ts

@@ -1486,6 +1486,7 @@ export class Session extends EventDispatcher {
             const recorder = queryParams['recorder'];
             const webrtcStatsInterval = queryParams['webrtcStatsInterval'];
             const sendBrowserLogs = queryParams['sendBrowserLogs'];
+            const edition = queryParams['edition'];
 
             if (!!secret) {
                 this.openvidu.secret = secret;
@@ -1500,6 +1501,7 @@ export class Session extends EventDispatcher {
                 this.openvidu.sendBrowserLogs = sendBrowserLogs;
             }
             this.openvidu.isPro = !!webrtcStatsInterval && !!sendBrowserLogs;
+            this.openvidu.isEnterprise = edition === 'enterprise';
 
             this.openvidu.wsUri = 'wss://' + url.host + '/openvidu';
             this.openvidu.httpUri = 'https://' + url.host;

openvidu-browser/src/OpenVidu/Stream.ts

@@ -17,6 +17,7 @@
 
 import { Connection } from './Connection';
 import { Filter } from './Filter';
+import { Publisher } from './Publisher';
 import { Session } from './Session';
 import { StreamManager } from './StreamManager';
 import { Subscriber } from './Subscriber';
@@ -33,6 +34,8 @@ import { TypeOfVideo } from '../OpenViduInternal/Enums/TypeOfVideo';
 import { OpenViduLogger } from '../OpenViduInternal/Logger/OpenViduLogger';
 import { PlatformUtils } from '../OpenViduInternal/Utils/Platform';
 
+import { v4 as uuidv4 } from 'uuid';
+
 /**
  * @hidden
  */
@@ -151,6 +154,9 @@ export class Stream {
 
     private isSubscribeToRemote = false;
 
+    private virtualBackgroundSourceElements: { videoClone: HTMLVideoElement, mediaStreamClone: MediaStream };
+    private virtualBackgroundSinkElements: { VB: any, video: HTMLVideoElement, canvas: HTMLCanvasElement };
+
     /**
      * @hidden
      */
@@ -308,22 +314,9 @@ export class Stream {
      * @returns A Promise (to which you can optionally subscribe to) that is resolved to the applied filter if success and rejected with an Error object if not
      */
     applyFilter(type: string, options: Object): Promise<Filter> {
-        return new Promise((resolve, reject) => {
+        return new Promise(async (resolve, reject) => {
 
-            if (!this.session.sessionConnected()) {
+            const resolveApplyFilter = (error) => {
-                return reject(this.session.notConnectedError());
-            }
-
-            logger.info('Applying filter to stream ' + this.streamId);
-            options = options != null ? options : {};
-            let optionsString = options;
-            if (typeof optionsString !== 'string') {
-                optionsString = JSON.stringify(optionsString);
-            }
-            this.session.openvidu.sendRequest(
-                'applyFilter',
-                { streamId: this.streamId, type, options: optionsString },
-                (error, response) => {
                 if (error) {
                     logger.error('Error applying filter for Stream ' + this.streamId, error);
                     if (error.code === 401) {
@@ -341,7 +334,112 @@ export class Stream {
                     return resolve(this.filter);
                 }
             }
+
+            if (type === 'VB:blur') {
+
+                // Client filters
+
+                if (!this.session.openvidu.httpUri) {
+                    return reject(this.session.notConnectedError());
+                }
+                if (!this.session.openvidu.isEnterprise) {
+                    return reject(new OpenViduError(OpenViduErrorName.OPENVIDU_EDITION_NOT_SUPPORTED, 'OpenVidu Virtual Background API is part of OpenVidu Enterprise edition'));
+                }
+                if (!this.hasVideo) {
+                    return reject(new OpenViduError(OpenViduErrorName.NO_VIDEO_TRACK, 'The Virtual Background filter requires a video track to be applied'));
+                }
+                if (!this.mediaStream || this.streamManager.videos.length === 0) {
+                    return reject(new OpenViduError(OpenViduErrorName.STREAM_MANAGER_HAS_NO_VIDEO_ELEMENT, 'The StreamManager requires some video element to be attached to it in order to apply a Virtual Background filter'));
+                }
+
+                logger.info('Applying client filter to stream ' + this.streamId);
+
+                const afterScriptLoaded = async () => {
+                    try {
+                        const id = this.streamId + '_' + uuidv4();
+                        const mediaStreamClone = this.mediaStream!.clone();
+                        const videoClone = this.streamManager.videos[0].video.cloneNode(false) as HTMLVideoElement;
+                        videoClone.id = 'source_video_' + id;
+                        videoClone.srcObject = mediaStreamClone;
+                        this.virtualBackgroundSourceElements = { videoClone, mediaStreamClone };
+
+                        // @ts-ignore
+                        VirtualBackground.VirtualBackground.hideHtmlElement(videoClone, false);
+                        // @ts-ignore
+                        VirtualBackground.VirtualBackground.appendHtmlElementToHiddenContainer(videoClone);
+
+                        await videoClone.play();
+
+                        // @ts-ignore
+                        const VB = new VirtualBackground.VirtualBackground({
+                            id,
+                            openviduServerUrl: new URL(this.session.openvidu.httpUri),
+                            inputVideo: videoClone,
+                            inputResolution: '160x96',
+                            outputFramerate: 30
+                        });
+                        const response: { video: HTMLVideoElement, canvas: HTMLCanvasElement } = await VB.backgroundBlur({
+                            maskRadius: 0.1,
+                            backgroundCoverage: 0.6,
+                            lightWrapping: 0.3
+                        });
+                        this.virtualBackgroundSinkElements = { VB, ...response };
+
+                        videoClone.style.display = 'none';
+
+                        if (this.streamManager.remote) {
+                            this.streamManager.replaceTrackInMediaStream((this.virtualBackgroundSinkElements.video.srcObject as MediaStream).getVideoTracks()[0]);
+                        } else {
+                            (this.streamManager as Publisher).replaceTrack((this.virtualBackgroundSinkElements.video.srcObject as MediaStream).getVideoTracks()[0]);
+                        }
+
+                        resolveApplyFilter(undefined);
+
+                    } catch (error) {
+                        resolveApplyFilter(error);
+                    }
+                }
+
+                // @ts-ignore
+                if (typeof VirtualBackground === "undefined") {
+                    let script: HTMLScriptElement = document.createElement("script");
+                    script.type = "text/javascript";
+                    script.src = this.session.openvidu.httpUri + '/virtual-background/openvidu-virtual-background.js';
+                    script.onload = async () => {
+                        await afterScriptLoaded();
+                        resolve(new Filter(type, options));
+                    };
+                    document.body.appendChild(script);
+                } else {
+                    afterScriptLoaded()
+                        .then(() => resolve(new Filter(type, options)))
+                        .catch(error => reject(error));
+                }
+
+            } else {
+
+                // Server filters
+
+                if (!this.session.sessionConnected()) {
+                    return reject(this.session.notConnectedError());
+                }
+
+                logger.info('Applying server filter to stream ' + this.streamId);
+                options = options != null ? options : {};
+                let optionsString = options;
+                if (typeof optionsString !== 'string') {
+                    optionsString = JSON.stringify(optionsString);
+                }
+                this.session.openvidu.sendRequest(
+                    'applyFilter',
+                    { streamId: this.streamId, type, options: optionsString },
+                    (error, response) => {
+                        resolveApplyFilter(error);
+                    }
                 );
+
+            }
+
         });
     }
 
@@ -351,17 +449,9 @@ export class Stream {
      * @returns A Promise (to which you can optionally subscribe to) that is resolved if the previously applied filter has been successfully removed and rejected with an Error object in other case
      */
     removeFilter(): Promise<void> {
-        return new Promise((resolve, reject) => {
+        return new Promise(async (resolve, reject) => {
 
-            if (!this.session.sessionConnected()) {
+            const resolveRemoveFilter = (error) => {
-                return reject(this.session.notConnectedError());
-            }
-
-            logger.info('Removing filter of stream ' + this.streamId);
-            this.session.openvidu.sendRequest(
-                'removeFilter',
-                { streamId: this.streamId },
-                (error, response) => {
                 if (error) {
                     logger.error('Error removing filter for Stream ' + this.streamId, error);
                     if (error.code === 401) {
@@ -378,7 +468,52 @@ export class Stream {
                     return resolve();
                 }
             }
+
+            if (!!this.filter && this.filter?.type.startsWith('VB:')) {
+
+                // Client filters
+
+                try {
+
+                    this.virtualBackgroundSinkElements.VB.cleanUp();
+                    const parent = this.virtualBackgroundSourceElements.videoClone.parentElement;
+                    this.virtualBackgroundSourceElements.videoClone.remove();
+                    if (parent!.children.length === 0) {
+                        // @ts-ignore
+                        VirtualBackground.VirtualBackground.removeHiddenContainer();
+                    }
+
+                    if (this.streamManager.remote) {
+                        await this.streamManager.replaceTrackInMediaStream(this.virtualBackgroundSourceElements.mediaStreamClone.getVideoTracks()[0]);
+                    } else {
+                        await (this.streamManager as Publisher).replaceTrack(this.virtualBackgroundSourceElements.mediaStreamClone.getVideoTracks()[0]);
+                    }
+
+                    return resolveRemoveFilter(undefined);
+
+                } catch (error) {
+                    return resolveRemoveFilter(error);
+                }
+
+            } else {
+
+                // Server filters
+
+                if (!this.session.sessionConnected()) {
+                    return reject(this.session.notConnectedError());
+                }
+
+                logger.info('Removing filter of stream ' + this.streamId);
+                this.session.openvidu.sendRequest(
+                    'removeFilter',
+                    { streamId: this.streamId },
+                    (error, response) => {
+                        return resolveRemoveFilter(error);
+                    }
                 );
+
+            }
+
         });
     }
 

openvidu-browser/src/OpenVidu/StreamManager.ts

@@ -47,7 +47,7 @@ let platform: PlatformUtils;
  *
  * See available event listeners at [[StreamManagerEventMap]].
  */
-export class StreamManager extends EventDispatcher {
+export abstract class StreamManager extends EventDispatcher {
 
     /**
      * The Stream represented in the DOM by the Publisher/Subscriber
@@ -526,6 +526,11 @@ export class StreamManager extends EventDispatcher {
         this.deactivateStreamPlayingEventExceptionTimeout();
     }
 
+    /**
+     * @hidden
+     */
+    abstract replaceTrackInMediaStream(track: MediaStreamTrack): Promise<void>;
+
     /* Private methods */
 
     protected pushNewStreamManagerVideo(streamManagerVideo: StreamManagerVideo) {

openvidu-browser/src/OpenVidu/Subscriber.ts

@@ -73,4 +73,23 @@ export class Subscriber extends StreamManager {
         return this;
     }
 
+    /* Hidden methods */
+
+    /**
+     * @hidden
+     */
+    async replaceTrackInMediaStream(track: MediaStreamTrack): Promise<void> {
+        const mediaStream: MediaStream = this.stream.getMediaStream();
+        let removedTrack: MediaStreamTrack;
+        if (track.kind === 'video') {
+            removedTrack = mediaStream.getVideoTracks()[0];
+            this.stream.lastVideoTrackConstraints = track.getConstraints();
+        } else {
+            removedTrack = mediaStream.getAudioTracks()[0];
+        }
+        mediaStream.removeTrack(removedTrack);
+        removedTrack.stop();
+        mediaStream.addTrack(track);
+    }
+
 }

openvidu-browser/src/OpenViduInternal/Enums/OpenViduError.ts

@@ -107,6 +107,21 @@ export enum OpenViduErrorName {
      */
     OPENVIDU_NOT_CONNECTED = 'OPENVIDU_NOT_CONNECTED',
 
+    /**
+     * The action performed is not supported for this OpenVidu edition.
+     */
+    OPENVIDU_EDITION_NOT_SUPPORTED = 'OPENVIDU_EDITION_NOT_SUPPORTED',
+
+    /**
+     * The action performed requires a video track to be present.
+     */
+    NO_VIDEO_TRACK = 'NO_VIDEO_TRACK',
+
+    /**
+     * The action performed requires some video element to be attached to the [[StreamManager]].
+     */
+    STREAM_MANAGER_HAS_NO_VIDEO_ELEMENT = 'STREAM_MANAGER_HAS_NO_VIDEO_ELEMENT',
+
     /**
      * Generic error
      */