ambiser
/
openvidu
mirror of https://github.com/OpenVidu/openvidu.git
8
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

openvidu-browser: MVC Virtual Background

pull/713/head
pabloFuente 2022-04-06 13:10:06 +02:00
parent 4c2ab10e07
commit 841db74c75
7 changed files with 272 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
openvidu-browser/src/OpenVidu/OpenVidu.ts
View File

@ -133,6 +133,10 @@ export class OpenVidu {
* @hidden
*/
isPro: boolean = false;
/**
* @hidden
*/
isEnterprise: boolean = false;
/**
* @hidden
*/

82
openvidu-browser/src/OpenVidu/Publisher.ts
View File

@ -332,28 +332,6 @@ export class Publisher extends StreamManager {
* @returns A Promise (to which you can optionally subscribe to) that is resolved if the track was successfully replaced and rejected with an Error object in other case
*/
async replaceTrack(track: MediaStreamTrack): Promise<void> {
const replaceTrackInMediaStream = (): Promise<void> => {
return new Promise((resolve, reject) => {
const mediaStream: MediaStream = this.stream.displayMyRemote() ? this.stream.localMediaStreamWhenSubscribedToRemote! : this.stream.getMediaStream();
let removedTrack: MediaStreamTrack;
if (track.kind === 'video') {
removedTrack = mediaStream.getVideoTracks()[0];
this.stream.lastVideoTrackConstraints = track.getConstraints();
} else {
removedTrack = mediaStream.getAudioTracks()[0];
}
mediaStream.removeTrack(removedTrack);
removedTrack.stop();
mediaStream.addTrack(track);
if (track.kind === 'video' && this.stream.isLocalStreamPublished) {
this.openvidu.sendNewVideoDimensionsIfRequired(this, 'trackReplaced', 50, 30);
this.session.sendVideoData(this.stream.streamManager, 5, true, 5);
}
return resolve();
});
}
// Set field "enabled" of the new track to the previous value
const trackOriginalEnabledValue: boolean = track.enabled;
if (track.kind === 'video') {
@ -366,10 +344,10 @@ export class Publisher extends StreamManager {
// Only if the Publisher has been published is necessary to call native Web API RTCRtpSender.replaceTrack
// If it has not been published yet, replacing it on the MediaStream object is enough
await this.replaceTrackInRtcRtpSender(track);
return await replaceTrackInMediaStream();
return await this.replaceTrackInMediaStream(track);
} else {
// Publisher not published. Simply replace the track on the local MediaStream
return await replaceTrackInMediaStream();
return await this.replaceTrackInMediaStream(track);
}
} catch (error) {
track.enabled = trackOriginalEnabledValue;
@ -751,6 +729,26 @@ export class Publisher extends StreamManager {
this.videoReference.srcObject = mediaStream;
}
/**
* @hidden
*/
async replaceTrackInMediaStream(track: MediaStreamTrack): Promise<void> {
const mediaStream: MediaStream = this.stream.displayMyRemote() ? this.stream.localMediaStreamWhenSubscribedToRemote! : this.stream.getMediaStream();
let removedTrack: MediaStreamTrack;
if (track.kind === 'video') {
removedTrack = mediaStream.getVideoTracks()[0];
this.stream.lastVideoTrackConstraints = track.getConstraints();
} else {
removedTrack = mediaStream.getAudioTracks()[0];
}
mediaStream.removeTrack(removedTrack);
removedTrack.stop();
mediaStream.addTrack(track);
if (track.kind === 'video' && this.stream.isLocalStreamPublished) {
this.openvidu.sendNewVideoDimensionsIfRequired(this, 'trackReplaced', 50, 30);
this.session.sendVideoData(this.stream.streamManager, 5, true, 5);
}
}
/* Private methods */
@ -768,27 +766,23 @@ export class Publisher extends StreamManager {
}
}
private replaceTrackInRtcRtpSender(track: MediaStreamTrack): Promise<void> {
return new Promise((resolve, reject) => {
const senders: RTCRtpSender[] = this.stream.getRTCPeerConnection().getSenders();
let sender: RTCRtpSender | undefined;
if (track.kind === 'video') {
sender = senders.find(s => !!s.track && s.track.kind === 'video');
if (!sender) {
return reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object'));
}
} else if (track.kind === 'audio') {
sender = senders.find(s => !!s.track && s.track.kind === 'audio');
if (!sender) {
return reject(new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object'));
}
} else {
return reject(new Error('Unknown track kind ' + track.kind));
private async replaceTrackInRtcRtpSender(track: MediaStreamTrack): Promise<void> {
const senders: RTCRtpSender[] = this.stream.getRTCPeerConnection().getSenders();
let sender: RTCRtpSender | undefined;
if (track.kind === 'video') {
sender = senders.find(s => !!s.track && s.track.kind === 'video');
if (!sender) {
throw new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object');
}
(sender as RTCRtpSender).replaceTrack(track)
.then(() => resolve())
.catch(error => reject(error));
});
} else if (track.kind === 'audio') {
sender = senders.find(s => !!s.track && s.track.kind === 'audio');
if (!sender) {
throw new Error('There\'s no replaceable track for that kind of MediaStreamTrack in this Publisher object');
}
} else {
throw new Error('Unknown track kind ' + track.kind);
}
await (sender as RTCRtpSender).replaceTrack(track);
}
}

2
openvidu-browser/src/OpenVidu/Session.ts
View File

@ -1486,6 +1486,7 @@ export class Session extends EventDispatcher {
const recorder = queryParams['recorder'];
const webrtcStatsInterval = queryParams['webrtcStatsInterval'];
const sendBrowserLogs = queryParams['sendBrowserLogs'];
const edition = queryParams['edition'];
if (!!secret) {
this.openvidu.secret = secret;
@ -1500,6 +1501,7 @@ export class Session extends EventDispatcher {
this.openvidu.sendBrowserLogs = sendBrowserLogs;
}
this.openvidu.isPro = !!webrtcStatsInterval && !!sendBrowserLogs;
this.openvidu.isEnterprise = edition === 'enterprise';
this.openvidu.wsUri = 'wss://' + url.host + '/openvidu';
this.openvidu.httpUri = 'https://' + url.host;

241
openvidu-browser/src/OpenVidu/Stream.ts
View File

@ -17,6 +17,7 @@
import { Connection } from './Connection';
import { Filter } from './Filter';
import { Publisher } from './Publisher';
import { Session } from './Session';
import { StreamManager } from './StreamManager';
import { Subscriber } from './Subscriber';
@ -33,6 +34,8 @@ import { TypeOfVideo } from '../OpenViduInternal/Enums/TypeOfVideo';
import { OpenViduLogger } from '../OpenViduInternal/Logger/OpenViduLogger';
import { PlatformUtils } from '../OpenViduInternal/Utils/Platform';
import { v4 as uuidv4 } from 'uuid';
/**
* @hidden
*/
@ -151,6 +154,9 @@ export class Stream {
private isSubscribeToRemote = false;
private virtualBackgroundSourceElements: { videoClone: HTMLVideoElement, mediaStreamClone: MediaStream };
private virtualBackgroundSinkElements: { VB: any, video: HTMLVideoElement, canvas: HTMLCanvasElement };
/**
* @hidden
*/
@ -308,40 +314,132 @@ export class Stream {
* @returns A Promise (to which you can optionally subscribe to) that is resolved to the applied filter if success and rejected with an Error object if not
*/
applyFilter(type: string, options: Object): Promise<Filter> {
return new Promise((resolve, reject) => {
return new Promise(async (resolve, reject) => {
if (!this.session.sessionConnected()) {
return reject(this.session.notConnectedError());
}
logger.info('Applying filter to stream ' + this.streamId);
options = options != null ? options : {};
let optionsString = options;
if (typeof optionsString !== 'string') {
optionsString = JSON.stringify(optionsString);
}
this.session.openvidu.sendRequest(
'applyFilter',
{ streamId: this.streamId, type, options: optionsString },
(error, response) => {
if (error) {
logger.error('Error applying filter for Stream ' + this.streamId, error);
if (error.code === 401) {
return reject(new OpenViduError(OpenViduErrorName.OPENVIDU_PERMISSION_DENIED, "You don't have permissions to apply a filter"));
} else {
return reject(error);
}
const resolveApplyFilter = (error) => {
if (error) {
logger.error('Error applying filter for Stream ' + this.streamId, error);
if (error.code === 401) {
return reject(new OpenViduError(OpenViduErrorName.OPENVIDU_PERMISSION_DENIED, "You don't have permissions to apply a filter"));
} else {
logger.info('Filter successfully applied on Stream ' + this.streamId);
const oldValue: Filter = this.filter!;
this.filter = new Filter(type, options);
this.filter.stream = this;
this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, this, 'filter', this.filter, oldValue, 'applyFilter')]);
this.streamManager.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.streamManager, this, 'filter', this.filter, oldValue, 'applyFilter')]);
return resolve(this.filter);
return reject(error);
}
} else {
logger.info('Filter successfully applied on Stream ' + this.streamId);
const oldValue: Filter = this.filter!;
this.filter = new Filter(type, options);
this.filter.stream = this;
this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, this, 'filter', this.filter, oldValue, 'applyFilter')]);
this.streamManager.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.streamManager, this, 'filter', this.filter, oldValue, 'applyFilter')]);
return resolve(this.filter);
}
}
if (type === 'VB:blur') {
// Client filters
if (!this.session.openvidu.httpUri) {
return reject(this.session.notConnectedError());
}
if (!this.session.openvidu.isEnterprise) {
return reject(new OpenViduError(OpenViduErrorName.OPENVIDU_EDITION_NOT_SUPPORTED, 'OpenVidu Virtual Background API is part of OpenVidu Enterprise edition'));
}
if (!this.hasVideo) {
return reject(new OpenViduError(OpenViduErrorName.NO_VIDEO_TRACK, 'The Virtual Background filter requires a video track to be applied'));
}
if (!this.mediaStream || this.streamManager.videos.length === 0) {
return reject(new OpenViduError(OpenViduErrorName.STREAM_MANAGER_HAS_NO_VIDEO_ELEMENT, 'The StreamManager requires some video element to be attached to it in order to apply a Virtual Background filter'));
}
logger.info('Applying client filter to stream ' + this.streamId);
const afterScriptLoaded = async () => {
try {
const id = this.streamId + '_' + uuidv4();
const mediaStreamClone = this.mediaStream!.clone();
const videoClone = this.streamManager.videos[0].video.cloneNode(false) as HTMLVideoElement;
videoClone.id = 'source_video_' + id;
videoClone.srcObject = mediaStreamClone;
this.virtualBackgroundSourceElements = { videoClone, mediaStreamClone };
// @ts-ignore
VirtualBackground.VirtualBackground.hideHtmlElement(videoClone, false);
// @ts-ignore
VirtualBackground.VirtualBackground.appendHtmlElementToHiddenContainer(videoClone);
await videoClone.play();
// @ts-ignore
const VB = new VirtualBackground.VirtualBackground({
id,
openviduServerUrl: new URL(this.session.openvidu.httpUri),
inputVideo: videoClone,
inputResolution: '160x96',
outputFramerate: 30
});
const response: { video: HTMLVideoElement, canvas: HTMLCanvasElement } = await VB.backgroundBlur({
maskRadius: 0.1,
backgroundCoverage: 0.6,
lightWrapping: 0.3
});
this.virtualBackgroundSinkElements = { VB, ...response };
videoClone.style.display = 'none';
if (this.streamManager.remote) {
this.streamManager.replaceTrackInMediaStream((this.virtualBackgroundSinkElements.video.srcObject as MediaStream).getVideoTracks()[0]);
} else {
(this.streamManager as Publisher).replaceTrack((this.virtualBackgroundSinkElements.video.srcObject as MediaStream).getVideoTracks()[0]);
}
resolveApplyFilter(undefined);
} catch (error) {
resolveApplyFilter(error);
}
}
);
// @ts-ignore
if (typeof VirtualBackground === "undefined") {
let script: HTMLScriptElement = document.createElement("script");
script.type = "text/javascript";
script.src = this.session.openvidu.httpUri + '/virtual-background/openvidu-virtual-background.js';
script.onload = async () => {
await afterScriptLoaded();
resolve(new Filter(type, options));
};
document.body.appendChild(script);
} else {
afterScriptLoaded()
.then(() => resolve(new Filter(type, options)))
.catch(error => reject(error));
}
} else {
// Server filters
if (!this.session.sessionConnected()) {
return reject(this.session.notConnectedError());
}
logger.info('Applying server filter to stream ' + this.streamId);
options = options != null ? options : {};
let optionsString = options;
if (typeof optionsString !== 'string') {
optionsString = JSON.stringify(optionsString);
}
this.session.openvidu.sendRequest(
'applyFilter',
{ streamId: this.streamId, type, options: optionsString },
(error, response) => {
resolveApplyFilter(error);
}
);
}
});
}
@ -351,34 +449,71 @@ export class Stream {
* @returns A Promise (to which you can optionally subscribe to) that is resolved if the previously applied filter has been successfully removed and rejected with an Error object in other case
*/
removeFilter(): Promise<void> {
return new Promise((resolve, reject) => {
return new Promise(async (resolve, reject) => {
if (!this.session.sessionConnected()) {
return reject(this.session.notConnectedError());
const resolveRemoveFilter = (error) => {
if (error) {
logger.error('Error removing filter for Stream ' + this.streamId, error);
if (error.code === 401) {
return reject(new OpenViduError(OpenViduErrorName.OPENVIDU_PERMISSION_DENIED, "You don't have permissions to remove a filter"));
} else {
return reject(error);
}
} else {
logger.info('Filter successfully removed from Stream ' + this.streamId);
const oldValue = this.filter!;
delete this.filter;
this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, this, 'filter', this.filter!, oldValue, 'applyFilter')]);
this.streamManager.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.streamManager, this, 'filter', this.filter!, oldValue, 'applyFilter')]);
return resolve();
}
}
logger.info('Removing filter of stream ' + this.streamId);
this.session.openvidu.sendRequest(
'removeFilter',
{ streamId: this.streamId },
(error, response) => {
if (error) {
logger.error('Error removing filter for Stream ' + this.streamId, error);
if (error.code === 401) {
return reject(new OpenViduError(OpenViduErrorName.OPENVIDU_PERMISSION_DENIED, "You don't have permissions to remove a filter"));
} else {
return reject(error);
}
} else {
logger.info('Filter successfully removed from Stream ' + this.streamId);
const oldValue = this.filter!;
delete this.filter;
this.session.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.session, this, 'filter', this.filter!, oldValue, 'applyFilter')]);
this.streamManager.emitEvent('streamPropertyChanged', [new StreamPropertyChangedEvent(this.streamManager, this, 'filter', this.filter!, oldValue, 'applyFilter')]);
return resolve();
if (!!this.filter && this.filter?.type.startsWith('VB:')) {
// Client filters
try {
this.virtualBackgroundSinkElements.VB.cleanUp();
const parent = this.virtualBackgroundSourceElements.videoClone.parentElement;
this.virtualBackgroundSourceElements.videoClone.remove();
if (parent!.children.length === 0) {
// @ts-ignore
VirtualBackground.VirtualBackground.removeHiddenContainer();
}
if (this.streamManager.remote) {
await this.streamManager.replaceTrackInMediaStream(this.virtualBackgroundSourceElements.mediaStreamClone.getVideoTracks()[0]);
} else {
await (this.streamManager as Publisher).replaceTrack(this.virtualBackgroundSourceElements.mediaStreamClone.getVideoTracks()[0]);
}
return resolveRemoveFilter(undefined);
} catch (error) {
return resolveRemoveFilter(error);
}
);
} else {
// Server filters
if (!this.session.sessionConnected()) {
return reject(this.session.notConnectedError());
}
logger.info('Removing filter of stream ' + this.streamId);
this.session.openvidu.sendRequest(
'removeFilter',
{ streamId: this.streamId },
(error, response) => {
return resolveRemoveFilter(error);
}
);
}
});
}

7
openvidu-browser/src/OpenVidu/StreamManager.ts
View File

@ -47,7 +47,7 @@ let platform: PlatformUtils;
*
* See available event listeners at [[StreamManagerEventMap]].
*/
export class StreamManager extends EventDispatcher {
export abstract class StreamManager extends EventDispatcher {
/**
* The Stream represented in the DOM by the Publisher/Subscriber
@ -526,6 +526,11 @@ export class StreamManager extends EventDispatcher {
this.deactivateStreamPlayingEventExceptionTimeout();
}
/**
* @hidden
*/
abstract replaceTrackInMediaStream(track: MediaStreamTrack): Promise<void>;
/* Private methods */
protected pushNewStreamManagerVideo(streamManagerVideo: StreamManagerVideo) {

19
openvidu-browser/src/OpenVidu/Subscriber.ts
View File

@ -73,4 +73,23 @@ export class Subscriber extends StreamManager {
return this;
}
/* Hidden methods */
/**
* @hidden
*/
async replaceTrackInMediaStream(track: MediaStreamTrack): Promise<void> {
const mediaStream: MediaStream = this.stream.getMediaStream();
let removedTrack: MediaStreamTrack;
if (track.kind === 'video') {
removedTrack = mediaStream.getVideoTracks()[0];
this.stream.lastVideoTrackConstraints = track.getConstraints();
} else {
removedTrack = mediaStream.getAudioTracks()[0];
}
mediaStream.removeTrack(removedTrack);
removedTrack.stop();
mediaStream.addTrack(track);
}
}

15
openvidu-browser/src/OpenViduInternal/Enums/OpenViduError.ts
View File

@ -107,6 +107,21 @@ export enum OpenViduErrorName {
*/
OPENVIDU_NOT_CONNECTED = 'OPENVIDU_NOT_CONNECTED',
/**
* The action performed is not supported for this OpenVidu edition.
*/
OPENVIDU_EDITION_NOT_SUPPORTED = 'OPENVIDU_EDITION_NOT_SUPPORTED',
/**
* The action performed requires a video track to be present.
*/
NO_VIDEO_TRACK = 'NO_VIDEO_TRACK',
/**
* The action performed requires some video element to be attached to the [[StreamManager]].
*/
STREAM_MANAGER_HAS_NO_VIDEO_ELEMENT = 'STREAM_MANAGER_HAS_NO_VIDEO_ELEMENT',
/**
* Generic error
*/