From 6fe6b52dcfb01ecb8831a576140205922aebab20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20=C5=A0korpil?= Date: Mon, 18 Dec 2017 19:28:12 +0100 Subject: [PATCH 1/3] Events MediaFlowInChange and MediaFlowOutChange are now sent to clients --- .../client/internal/ProtocolElements.java | 10 ++++++ .../openvidu/server/core/api/RoomHandler.java | 25 +++++++++++++++ .../DefaultNotificationRoomHandler.java | 32 +++++++++++++++++-- .../server/core/internal/Participant.java | 21 +++++++----- .../openvidu/server/core/internal/Room.java | 17 ++++++---- 5 files changed, 88 insertions(+), 17 deletions(-) diff --git a/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java b/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java index e33621fa..6cf613f8 100644 --- a/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java +++ b/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java @@ -112,5 +112,15 @@ public class ProtocolElements { public static final String ICECANDIDATE_SDPMID_PARAM = "sdpMid"; public static final String ICECANDIDATE_SDPMLINEINDEX_PARAM = "sdpMLineIndex"; + public static final String MEDIAFLOWINCHANGE_METHOD = "onMediaFlowInChange"; + public static final String MEDIAFLOWINCHANGE_MEDIATYPE_PARAM = "mediaType"; + public static final String MEDIAFLOWINCHANGE_NEWSTATE_PARAM = "newState"; + public static final String MEDIAFLOWINCHANGE_USER_PARAM = "id"; + + public static final String MEDIAFLOWOUTCHANGE_METHOD = "onMediaFlowOutChange"; + public static final String MEDIAFLOWOUTCHANGE_MEDIATYPE_PARAM = "mediaType"; + public static final String MEDIAFLOWOUTCHANGE_NEWSTATE_PARAM = "newState"; + public static final String MEDIAFLOWOUTCHANGE_USER_PARAM = "id"; + public static final String CUSTOM_NOTIFICATION = "custonNotification"; } diff --git a/openvidu-server/src/main/java/io/openvidu/server/core/api/RoomHandler.java b/openvidu-server/src/main/java/io/openvidu/server/core/api/RoomHandler.java index 589fbd9a..1b63e293 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/core/api/RoomHandler.java +++ b/openvidu-server/src/main/java/io/openvidu/server/core/api/RoomHandler.java @@ -16,12 +16,17 @@ package io.openvidu.server.core.api; +import java.util.Collection; +import java.util.HashMap; import java.util.Set; +import io.openvidu.server.core.api.pojo.UserParticipant; import org.kurento.client.IceCandidate; import io.openvidu.server.InfoHandler; import io.openvidu.server.core.internal.Participant; +import org.kurento.client.MediaFlowState; +import org.kurento.client.MediaType; /** * Handler for events triggered from media objects. @@ -73,6 +78,26 @@ public interface RoomHandler { */ void updateFilter(String roomName, Participant participant, String filterId, String state); + /** + * Called as a result of started or interrupted media flow. All participants should be notified. + * @param participantId + * @param mediaType + * @param newState + * @param participants + */ + void onMediaFlowInChange(String roomName, String participantId, MediaType mediaType, + MediaFlowState newState, Collection participants); + + /** + * Called as a result of started or interrupted media flow. All participants should be notified. + * @param participantId + * @param mediaType + * @param newState + * @param participants + */ + void onMediaFlowOutChange(String roomName, String participantId, MediaType mediaType, + MediaFlowState newState, Collection participants); + /** * Called to get the next state of a filter when requested by a call to updateFilter * diff --git a/openvidu-server/src/main/java/io/openvidu/server/core/internal/DefaultNotificationRoomHandler.java b/openvidu-server/src/main/java/io/openvidu/server/core/internal/DefaultNotificationRoomHandler.java index 1a7e8b76..a8c23774 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/core/internal/DefaultNotificationRoomHandler.java +++ b/openvidu-server/src/main/java/io/openvidu/server/core/internal/DefaultNotificationRoomHandler.java @@ -16,12 +16,12 @@ package io.openvidu.server.core.internal; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import org.kurento.client.IceCandidate; +import org.kurento.client.MediaFlowState; +import org.kurento.client.MediaType; import org.springframework.beans.factory.annotation.Autowired; import com.google.gson.JsonArray; @@ -306,6 +306,32 @@ public class DefaultNotificationRoomHandler implements NotificationRoomHandler { ProtocolElements.PARTICIPANTEVICTED_METHOD, new JsonObject()); } + @Override + public void onMediaFlowInChange(String roomName, String participantId, MediaType mediaType, + MediaFlowState newState, Collection participants) { + JsonObject notifParams = new JsonObject(); + notifParams.addProperty(ProtocolElements.MEDIAFLOWINCHANGE_USER_PARAM, participantId); + notifParams.addProperty(ProtocolElements.MEDIAFLOWINCHANGE_MEDIATYPE_PARAM, mediaType.toString()); + notifParams.addProperty(ProtocolElements.MEDIAFLOWINCHANGE_NEWSTATE_PARAM, newState.toString()); + for (Participant participant : participants) { + notifService.sendNotification(participant.getId(), + ProtocolElements.MEDIAFLOWINCHANGE_METHOD, notifParams); + } + } + + @Override + public void onMediaFlowOutChange(String roomName, String participantId, MediaType mediaType, + MediaFlowState newState, Collection participants) { + JsonObject notifParams = new JsonObject(); + notifParams.addProperty(ProtocolElements.MEDIAFLOWOUTCHANGE_USER_PARAM, participantId); + notifParams.addProperty(ProtocolElements.MEDIAFLOWOUTCHANGE_MEDIATYPE_PARAM, mediaType.toString()); + notifParams.addProperty(ProtocolElements.MEDIAFLOWOUTCHANGE_NEWSTATE_PARAM, newState.toString()); + for (Participant participant : participants) { + notifService.sendNotification(participant.getId(), + ProtocolElements.MEDIAFLOWOUTCHANGE_METHOD, notifParams); + } + } + // ------------ EVENTS FROM ROOM HANDLER ----- @Override diff --git a/openvidu-server/src/main/java/io/openvidu/server/core/internal/Participant.java b/openvidu-server/src/main/java/io/openvidu/server/core/internal/Participant.java index 89452f5c..76f0c81f 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/core/internal/Participant.java +++ b/openvidu-server/src/main/java/io/openvidu/server/core/internal/Participant.java @@ -23,14 +23,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import org.kurento.client.Continuation; -import org.kurento.client.ErrorEvent; -import org.kurento.client.Filter; -import org.kurento.client.IceCandidate; -import org.kurento.client.MediaElement; -import org.kurento.client.MediaPipeline; -import org.kurento.client.MediaType; -import org.kurento.client.SdpEndpoint; +import org.kurento.client.*; import org.kurento.client.internal.server.KurentoServerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -498,6 +491,16 @@ public class Participant { room.sendMediaError(id, desc); } + public void sendMediaFlowInChange(MediaFlowInStateChangeEvent event) { + log.warn("PARTICIPANT {}: Media flow in: {}", name, event.getState()); + room.sendMediaFlowInChange(name, event.getMediaType(),event.getState()); + } + + public void sendMediaFlowOutChange(MediaFlowOutStateChangeEvent event) { + log.warn("PARTICIPANT {}: Media flow out: {}", name, event.getState()); + room.sendMediaFlowOutChange(name, event.getMediaType(),event.getState()); + } + private void releasePublisherEndpoint() { if (publisher != null && publisher.getEndpoint() != null) { this.streaming = false; @@ -638,6 +641,7 @@ public class Participant { System.out.println(msg2); this.infoHandler.sendInfo(msg1); this.infoHandler.sendInfo(msg2); + this.sendMediaFlowInChange(event); }); endpoint.getWebEndpoint().addMediaFlowOutStateChangeListener((event) -> { @@ -662,6 +666,7 @@ public class Participant { System.out.println(msg2); this.infoHandler.sendInfo(msg1); this.infoHandler.sendInfo(msg2); + this.sendMediaFlowOutChange(event); }); endpoint.getWebEndpoint().addMediaSessionStartedListener((event) -> { diff --git a/openvidu-server/src/main/java/io/openvidu/server/core/internal/Room.java b/openvidu-server/src/main/java/io/openvidu/server/core/internal/Room.java index d5a1b807..0bccc50f 100644 --- a/openvidu-server/src/main/java/io/openvidu/server/core/internal/Room.java +++ b/openvidu-server/src/main/java/io/openvidu/server/core/internal/Room.java @@ -24,12 +24,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import org.kurento.client.Continuation; -import org.kurento.client.ErrorEvent; -import org.kurento.client.EventListener; -import org.kurento.client.IceCandidate; -import org.kurento.client.KurentoClient; -import org.kurento.client.MediaPipeline; +import org.kurento.client.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -236,6 +231,16 @@ public class Room { this.roomHandler.onIceCandidate(name, participantId, endpointName, candidate); } + public void sendMediaFlowInChange(String participantId, MediaType mediaType, + MediaFlowState newState){ + this.roomHandler.onMediaFlowInChange(name,participantId, mediaType, newState, participants.values()); + } + + public void sendMediaFlowOutChange(String participantId, MediaType mediaType, + MediaFlowState newState){ + this.roomHandler.onMediaFlowOutChange(name,participantId, mediaType, newState, participants.values()); + } + public void sendMediaError(String participantId, String description) { this.roomHandler.onMediaElementError(name, participantId, description); } From 511ffb0786081a229c458a51549e39e59d575abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20=C5=A0korpil?= Date: Mon, 18 Dec 2017 21:15:22 +0100 Subject: [PATCH 2/3] Renamed JsonRpc method mediaFlowChanged according to standards --- .../java/io/openvidu/client/internal/ProtocolElements.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java b/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java index 6cf613f8..cb0928cd 100644 --- a/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java +++ b/openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java @@ -112,12 +112,12 @@ public class ProtocolElements { public static final String ICECANDIDATE_SDPMID_PARAM = "sdpMid"; public static final String ICECANDIDATE_SDPMLINEINDEX_PARAM = "sdpMLineIndex"; - public static final String MEDIAFLOWINCHANGE_METHOD = "onMediaFlowInChange"; + public static final String MEDIAFLOWINCHANGE_METHOD = "mediaFlowInChange"; public static final String MEDIAFLOWINCHANGE_MEDIATYPE_PARAM = "mediaType"; public static final String MEDIAFLOWINCHANGE_NEWSTATE_PARAM = "newState"; public static final String MEDIAFLOWINCHANGE_USER_PARAM = "id"; - public static final String MEDIAFLOWOUTCHANGE_METHOD = "onMediaFlowOutChange"; + public static final String MEDIAFLOWOUTCHANGE_METHOD = "mediaFlowOutChange"; public static final String MEDIAFLOWOUTCHANGE_MEDIATYPE_PARAM = "mediaType"; public static final String MEDIAFLOWOUTCHANGE_NEWSTATE_PARAM = "newState"; public static final String MEDIAFLOWOUTCHANGE_USER_PARAM = "id"; From 4086744e156e94a180957214e35181acbac53f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0t=C4=9Bp=C3=A1n=20=C5=A0korpil?= Date: Fri, 22 Dec 2017 14:38:29 +0100 Subject: [PATCH 3/3] Added madia flow out chnage event distribution in openvidu-browser --- openvidu-browser/ts/OpenVidu/Publisher.ts | 18 +++++++++++++++ .../ts/OpenViduInternal/OpenViduInternal.ts | 15 +++++++++++- .../ts/OpenViduInternal/SessionInternal.ts | 15 ++++++++++++ .../ts/OpenViduInternal/Stream.ts | 23 +++++++++++++------ openvidu-node-client/lib/OpenVidu.js | 2 +- openvidu-node-client/lib/Session.js | 2 +- openvidu-node-client/lib/TokenOptions.js | 4 ++-- 7 files changed, 67 insertions(+), 12 deletions(-) diff --git a/openvidu-browser/ts/OpenVidu/Publisher.ts b/openvidu-browser/ts/OpenVidu/Publisher.ts index fd398232..8cc67062 100644 --- a/openvidu-browser/ts/OpenVidu/Publisher.ts +++ b/openvidu-browser/ts/OpenVidu/Publisher.ts @@ -20,6 +20,8 @@ export class Publisher { stream: Stream; session: Session; //Initialized by Session.publish(Publisher) isScreenRequested: boolean = false; + flowOutAudio: boolean = false; + flowOutVideo: boolean = false; constructor(stream: Stream, parentId: string, isScreenRequested: boolean) { this.stream = stream; @@ -34,6 +36,22 @@ export class Publisher { } }); + this.stream.addEventListener('media-flow-out-changed', (event) => { + let wasFlowing = this.flowOutAudio || this.flowOutVideo; + if(event.mediaType == "AUDIO"){ + this.flowOutAudio = event.newState == "FLOWING" + } + if(event.mediaType == "VIDEO"){ + this.flowOutVideo = event.newState == "FLOWING" + } + if(wasFlowing && !this.flowOutAudio && !this.flowOutVideo){ + this.ee.emitEvent('flowStopped'); + } + if (!wasFlowing && (this.flowOutAudio || this.flowOutVideo)) { + this.ee.emitEvent('flowStarted'); + } + }); + if (document.getElementById(parentId) != null) { this.element = document.getElementById(parentId)!!; } diff --git a/openvidu-browser/ts/OpenViduInternal/OpenViduInternal.ts b/openvidu-browser/ts/OpenViduInternal/OpenViduInternal.ts index a9b15b1b..99ef623b 100644 --- a/openvidu-browser/ts/OpenViduInternal/OpenViduInternal.ts +++ b/openvidu-browser/ts/OpenViduInternal/OpenViduInternal.ts @@ -193,7 +193,9 @@ export class OpenViduInternal { sendMessage: this.onNewMessage.bind(this), iceCandidate: this.iceCandidateEvent.bind(this), mediaError: this.onMediaError.bind(this), - custonNotification: this.customNotification.bind(this) + custonNotification: this.customNotification.bind(this), + mediaFlowInChange: this.onMediaFlowInChange.bind(this), + mediaFlowOutChange: this.onMediaFlowOutChange.bind(this), } }; @@ -294,6 +296,17 @@ export class OpenViduInternal { } } + private onMediaFlowOutChange(params) { + if (this.isRoomAvailable()) { + this.session.onMediaFlowOutChange(params); + } + } + + private onMediaFlowInChange(params) { + if (this.isRoomAvailable()) { + this.session.onMediaFlowInChange(params); + } + } setRpcParams(params: any) { this.rpcParams = params; diff --git a/openvidu-browser/ts/OpenViduInternal/SessionInternal.ts b/openvidu-browser/ts/OpenViduInternal/SessionInternal.ts index 4b366771..913c9500 100644 --- a/openvidu-browser/ts/OpenViduInternal/SessionInternal.ts +++ b/openvidu-browser/ts/OpenViduInternal/SessionInternal.ts @@ -460,6 +460,21 @@ export class SessionInternal { } } + onMediaFlowInChange(params) { + console.info("Media flow in change: " + JSON.stringify(params)); + } + + onMediaFlowOutChange(params) { + console.info("Media flow out change: " + JSON.stringify(params)); + for(let streamId in this.streams){ + this.streams[streamId].onMediaFlowOutChange({ + id: params.id, + mediaType: params.mediaType, + newState: params.newState + }); + } + } + /* * forced means the user was evicted, no need to send the 'leaveRoom' request */ diff --git a/openvidu-browser/ts/OpenViduInternal/Stream.ts b/openvidu-browser/ts/OpenViduInternal/Stream.ts index 331626e7..a1cca0ae 100644 --- a/openvidu-browser/ts/OpenViduInternal/Stream.ts +++ b/openvidu-browser/ts/OpenViduInternal/Stream.ts @@ -5,14 +5,15 @@ * * stream.hasAudio(); stream.hasVideo(); stream.hasData(); */ -import { Connection } from './Connection'; -import { SessionInternal } from './SessionInternal'; -import { OpenViduInternal, Callback } from './OpenViduInternal'; -import { OpenViduError, OpenViduErrorName } from './OpenViduError'; +import {Connection} from './Connection'; +import {SessionInternal} from './SessionInternal'; +import {OpenViduInternal, Callback} from './OpenViduInternal'; +import {OpenViduError, OpenViduErrorName} from './OpenViduError'; import EventEmitter = require('wolfy87-eventemitter'); import * as kurentoUtils from '../KurentoUtils/kurento-utils-js'; import * as adapter from 'webrtc-adapter'; + declare var navigator: any; declare var RTCSessionDescription: any; @@ -235,6 +236,13 @@ export class Stream { this.ee.removeAllListeners(eventName); } + onMediaFlowOutChange(event) { + if (event.id !== this.connection.connectionId) { + return; + } + this.ee.emit('media-flow-out-changed',event); + } + showSpinner(spinnerParentId: string) { let progress = document.createElement('div'); progress.id = 'progress-' + this.streamId; @@ -378,7 +386,8 @@ export class Stream { this.accessIsDenied = true; this.accessIsAllowed = false; let errorName: OpenViduErrorName; - let errorMessage = error.toString();; + let errorMessage = error.toString(); + ; if (!this.isScreenRequested) { errorName = this.sendVideo ? OpenViduErrorName.CAMERA_ACCESS_DENIED : OpenViduErrorName.MICROPHONE_ACCESS_DENIED; } else { @@ -437,7 +446,7 @@ export class Stream { doLoopback: this.displayMyRemote() || false, audioActive: this.sendAudio, videoActive: this.sendVideo, - typeOfVideo: ((this.sendVideo) ? ((this.isScreenRequested) ? 'SCREEN' :'CAMERA') : '') + typeOfVideo: ((this.sendVideo) ? ((this.isScreenRequested) ? 'SCREEN' : 'CAMERA') : '') }, (error, response) => { if (error) { console.error("Error on publishVideo: " + JSON.stringify(error)); @@ -578,7 +587,7 @@ export class Stream { if (this.wrStream.getAudioTracks()[0] != null) { - this.speechEvent = kurentoUtils.WebRtcPeer.hark(this.wrStream, { threshold: this.room.thresholdSpeaker }); + this.speechEvent = kurentoUtils.WebRtcPeer.hark(this.wrStream, {threshold: this.room.thresholdSpeaker}); this.speechEvent.on('speaking', () => { //this.room.addParticipantSpeaking(participantId); diff --git a/openvidu-node-client/lib/OpenVidu.js b/openvidu-node-client/lib/OpenVidu.js index 8a61fb6f..5a5cdf11 100644 --- a/openvidu-node-client/lib/OpenVidu.js +++ b/openvidu-node-client/lib/OpenVidu.js @@ -1,7 +1,7 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); var Session_1 = require("./Session"); -var OpenVidu = (function () { +var OpenVidu = /** @class */ (function () { function OpenVidu(urlOpenViduServer, secret) { this.urlOpenViduServer = urlOpenViduServer; this.secret = secret; diff --git a/openvidu-node-client/lib/Session.js b/openvidu-node-client/lib/Session.js index 8631339d..3b1b0b68 100644 --- a/openvidu-node-client/lib/Session.js +++ b/openvidu-node-client/lib/Session.js @@ -2,7 +2,7 @@ Object.defineProperty(exports, "__esModule", { value: true }); var OpenViduRole_1 = require("./OpenViduRole"); var https = require('https'); -var Session = (function () { +var Session = /** @class */ (function () { function Session(urlOpenViduServer, secret) { this.urlOpenViduServer = urlOpenViduServer; this.secret = secret; diff --git a/openvidu-node-client/lib/TokenOptions.js b/openvidu-node-client/lib/TokenOptions.js index 0c5bc9d1..17b187c4 100644 --- a/openvidu-node-client/lib/TokenOptions.js +++ b/openvidu-node-client/lib/TokenOptions.js @@ -1,6 +1,6 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -var TokenOptions = (function () { +var TokenOptions = /** @class */ (function () { function TokenOptions(data, role) { this.data = data; this.role = role; @@ -17,7 +17,7 @@ var TokenOptions = (function () { }()); exports.TokenOptions = TokenOptions; (function (TokenOptions) { - var Builder = (function () { + var Builder = /** @class */ (function () { function Builder() { } Builder.prototype.build = function () {