Merge 4086744e15 into 32b8a3b6e5

openvidu-browser/ts/OpenVidu/Publisher.ts

@@ -20,6 +20,8 @@ export class Publisher {
     stream: Stream;
     session: Session; //Initialized by Session.publish(Publisher)
     isScreenRequested: boolean = false;
+    flowOutAudio: boolean = false;
+    flowOutVideo: boolean = false;
 
     constructor(stream: Stream, parentId: string, isScreenRequested: boolean) {
         this.stream = stream;
@@ -34,6 +36,22 @@ export class Publisher {
             }
         });
 
+        this.stream.addEventListener('media-flow-out-changed', (event) => {
+            let wasFlowing = this.flowOutAudio || this.flowOutVideo;
+            if(event.mediaType == "AUDIO"){
+                this.flowOutAudio = event.newState == "FLOWING"
+            }
+            if(event.mediaType == "VIDEO"){
+                this.flowOutVideo = event.newState == "FLOWING"
+            }
+            if(wasFlowing && !this.flowOutAudio && !this.flowOutVideo){
+                this.ee.emitEvent('flowStopped');
+            }
+            if (!wasFlowing && (this.flowOutAudio || this.flowOutVideo)) {
+                this.ee.emitEvent('flowStarted');
+            }
+        });
+
         if (document.getElementById(parentId) != null) {
             this.element = document.getElementById(parentId)!!;
         }

openvidu-browser/ts/OpenViduInternal/OpenViduInternal.ts

@@ -193,7 +193,9 @@ export class OpenViduInternal {
                 sendMessage: this.onNewMessage.bind(this),
                 iceCandidate: this.iceCandidateEvent.bind(this),
                 mediaError: this.onMediaError.bind(this),
-                custonNotification: this.customNotification.bind(this)
+                custonNotification: this.customNotification.bind(this),
+                mediaFlowInChange: this.onMediaFlowInChange.bind(this),
+                mediaFlowOutChange: this.onMediaFlowOutChange.bind(this),
             }
         };
 
@@ -294,6 +296,17 @@ export class OpenViduInternal {
         }
     }
 
+    private onMediaFlowOutChange(params) {
+        if (this.isRoomAvailable()) {
+            this.session.onMediaFlowOutChange(params);
+        }
+    }
+
+    private onMediaFlowInChange(params) {
+        if (this.isRoomAvailable()) {
+            this.session.onMediaFlowInChange(params);
+        }
+    }
 
     setRpcParams(params: any) {
         this.rpcParams = params;

openvidu-browser/ts/OpenViduInternal/SessionInternal.ts

@@ -460,6 +460,21 @@ export class SessionInternal {
         }
     }
 
+    onMediaFlowInChange(params) {
+        console.info("Media flow in change: " + JSON.stringify(params));
+    }
+
+    onMediaFlowOutChange(params) {
+        console.info("Media flow out change: " + JSON.stringify(params));
+        for(let streamId in this.streams){
+            this.streams[streamId].onMediaFlowOutChange({
+                id: params.id,
+                mediaType: params.mediaType,
+                newState: params.newState
+            });
+        }
+    }
+
     /*
      * forced means the user was evicted, no need to send the 'leaveRoom' request
      */

openvidu-browser/ts/OpenViduInternal/Stream.ts

@@ -5,14 +5,15 @@
  *
  * stream.hasAudio(); stream.hasVideo(); stream.hasData();
  */
-import { Connection } from './Connection';
+import {Connection} from './Connection';
-import { SessionInternal } from './SessionInternal';
+import {SessionInternal} from './SessionInternal';
-import { OpenViduInternal, Callback } from './OpenViduInternal';
+import {OpenViduInternal, Callback} from './OpenViduInternal';
-import { OpenViduError, OpenViduErrorName } from './OpenViduError';
+import {OpenViduError, OpenViduErrorName} from './OpenViduError';
 import EventEmitter = require('wolfy87-eventemitter');
 import * as kurentoUtils from '../KurentoUtils/kurento-utils-js';
 
 import * as adapter from 'webrtc-adapter';
+
 declare var navigator: any;
 declare var RTCSessionDescription: any;
 
@@ -235,6 +236,13 @@ export class Stream {
         this.ee.removeAllListeners(eventName);
     }
 
+    onMediaFlowOutChange(event) {
+        if (event.id !== this.connection.connectionId) {
+            return;
+        }
+        this.ee.emit('media-flow-out-changed',event);
+    }
+
     showSpinner(spinnerParentId: string) {
         let progress = document.createElement('div');
         progress.id = 'progress-' + this.streamId;
@@ -378,7 +386,8 @@ export class Stream {
                 this.accessIsDenied = true;
                 this.accessIsAllowed = false;
                 let errorName: OpenViduErrorName;
-                let errorMessage = error.toString();;
+                let errorMessage = error.toString();
+                ;
                 if (!this.isScreenRequested) {
                     errorName = this.sendVideo ? OpenViduErrorName.CAMERA_ACCESS_DENIED : OpenViduErrorName.MICROPHONE_ACCESS_DENIED;
                 } else {
@@ -437,7 +446,7 @@ export class Stream {
             doLoopback: this.displayMyRemote() || false,
             audioActive: this.sendAudio,
             videoActive: this.sendVideo,
-            typeOfVideo: ((this.sendVideo) ? ((this.isScreenRequested) ? 'SCREEN' :'CAMERA') : '')
+            typeOfVideo: ((this.sendVideo) ? ((this.isScreenRequested) ? 'SCREEN' : 'CAMERA') : '')
         }, (error, response) => {
             if (error) {
                 console.error("Error on publishVideo: " + JSON.stringify(error));
@@ -578,7 +587,7 @@ export class Stream {
 
                     if (this.wrStream.getAudioTracks()[0] != null) {
 
-                        this.speechEvent = kurentoUtils.WebRtcPeer.hark(this.wrStream, { threshold: this.room.thresholdSpeaker });
+                        this.speechEvent = kurentoUtils.WebRtcPeer.hark(this.wrStream, {threshold: this.room.thresholdSpeaker});
 
                         this.speechEvent.on('speaking', () => {
                             //this.room.addParticipantSpeaking(participantId);

openvidu-client/src/main/java/io/openvidu/client/internal/ProtocolElements.java

@@ -112,5 +112,15 @@ public class ProtocolElements {
   public static final String ICECANDIDATE_SDPMID_PARAM = "sdpMid";
   public static final String ICECANDIDATE_SDPMLINEINDEX_PARAM = "sdpMLineIndex";
 
+  public static final String MEDIAFLOWINCHANGE_METHOD = "mediaFlowInChange";
+  public static final String MEDIAFLOWINCHANGE_MEDIATYPE_PARAM = "mediaType";
+  public static final String MEDIAFLOWINCHANGE_NEWSTATE_PARAM = "newState";
+  public static final String MEDIAFLOWINCHANGE_USER_PARAM = "id";
+
+  public static final String MEDIAFLOWOUTCHANGE_METHOD = "mediaFlowOutChange";
+  public static final String MEDIAFLOWOUTCHANGE_MEDIATYPE_PARAM = "mediaType";
+  public static final String MEDIAFLOWOUTCHANGE_NEWSTATE_PARAM = "newState";
+  public static final String MEDIAFLOWOUTCHANGE_USER_PARAM = "id";
+
   public static final String CUSTOM_NOTIFICATION = "custonNotification";
 }

openvidu-node-client/lib/OpenVidu.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 var Session_1 = require("./Session");
-var OpenVidu = (function () {
+var OpenVidu = /** @class */ (function () {
     function OpenVidu(urlOpenViduServer, secret) {
         this.urlOpenViduServer = urlOpenViduServer;
         this.secret = secret;

openvidu-node-client/lib/Session.js

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 var OpenViduRole_1 = require("./OpenViduRole");
 var https = require('https');
-var Session = (function () {
+var Session = /** @class */ (function () {
     function Session(urlOpenViduServer, secret) {
         this.urlOpenViduServer = urlOpenViduServer;
         this.secret = secret;

openvidu-node-client/lib/TokenOptions.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-var TokenOptions = (function () {
+var TokenOptions = /** @class */ (function () {
     function TokenOptions(data, role) {
         this.data = data;
         this.role = role;
@@ -17,7 +17,7 @@ var TokenOptions = (function () {
 }());
 exports.TokenOptions = TokenOptions;
 (function (TokenOptions) {
-    var Builder = (function () {
+    var Builder = /** @class */ (function () {
         function Builder() {
         }
         Builder.prototype.build = function () {

openvidu-server/src/main/java/io/openvidu/server/core/api/RoomHandler.java

@@ -16,12 +16,17 @@
 
 package io.openvidu.server.core.api;
 
+import java.util.Collection;
+import java.util.HashMap;
 import java.util.Set;
 
+import io.openvidu.server.core.api.pojo.UserParticipant;
 import org.kurento.client.IceCandidate;
 
 import io.openvidu.server.InfoHandler;
 import io.openvidu.server.core.internal.Participant;
+import org.kurento.client.MediaFlowState;
+import org.kurento.client.MediaType;
 
 /**
  * Handler for events triggered from media objects.
@@ -73,6 +78,26 @@ public interface RoomHandler {
    */
   void updateFilter(String roomName, Participant participant, String filterId, String state);
 
+  /**
+   * Called as a result of started or interrupted media flow. All participants should be notified.
+   * @param participantId
+   * @param mediaType
+   * @param newState
+   * @param participants
+   */
+  void onMediaFlowInChange(String roomName, String participantId, MediaType mediaType,
+                           MediaFlowState newState, Collection<Participant> participants);
+
+  /**
+   * Called as a result of started or interrupted media flow. All participants should be notified.
+   * @param participantId
+   * @param mediaType
+   * @param newState
+   * @param participants
+   */
+  void onMediaFlowOutChange(String roomName, String participantId, MediaType mediaType,
+                           MediaFlowState newState, Collection<Participant> participants);
+
   /**
    * Called to get the next state of a filter when requested by a call to updateFilter
    *

openvidu-server/src/main/java/io/openvidu/server/core/internal/DefaultNotificationRoomHandler.java

@@ -16,12 +16,12 @@
 
 package io.openvidu.server.core.internal;
 
-import java.util.HashSet;
+import java.util.*;
-import java.util.Optional;
-import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.kurento.client.IceCandidate;
+import org.kurento.client.MediaFlowState;
+import org.kurento.client.MediaType;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import com.google.gson.JsonArray;
@@ -306,6 +306,32 @@ public class DefaultNotificationRoomHandler implements NotificationRoomHandler {
         ProtocolElements.PARTICIPANTEVICTED_METHOD, new JsonObject());
   }
 
+  @Override
+  public void onMediaFlowInChange(String roomName, String participantId, MediaType mediaType,
+                                  MediaFlowState newState, Collection<Participant> participants) {
+    JsonObject notifParams = new JsonObject();
+    notifParams.addProperty(ProtocolElements.MEDIAFLOWINCHANGE_USER_PARAM, participantId);
+    notifParams.addProperty(ProtocolElements.MEDIAFLOWINCHANGE_MEDIATYPE_PARAM, mediaType.toString());
+    notifParams.addProperty(ProtocolElements.MEDIAFLOWINCHANGE_NEWSTATE_PARAM, newState.toString());
+    for (Participant participant : participants) {
+      notifService.sendNotification(participant.getId(),
+              ProtocolElements.MEDIAFLOWINCHANGE_METHOD, notifParams);
+    }
+  }
+
+  @Override
+  public void onMediaFlowOutChange(String roomName, String participantId, MediaType mediaType,
+                                  MediaFlowState newState, Collection<Participant> participants) {
+    JsonObject notifParams = new JsonObject();
+    notifParams.addProperty(ProtocolElements.MEDIAFLOWOUTCHANGE_USER_PARAM, participantId);
+    notifParams.addProperty(ProtocolElements.MEDIAFLOWOUTCHANGE_MEDIATYPE_PARAM, mediaType.toString());
+    notifParams.addProperty(ProtocolElements.MEDIAFLOWOUTCHANGE_NEWSTATE_PARAM, newState.toString());
+    for (Participant participant : participants) {
+      notifService.sendNotification(participant.getId(),
+              ProtocolElements.MEDIAFLOWOUTCHANGE_METHOD, notifParams);
+    }
+  }
+
   // ------------ EVENTS FROM ROOM HANDLER -----
 
   @Override

openvidu-server/src/main/java/io/openvidu/server/core/internal/Participant.java

@@ -23,14 +23,7 @@ import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
-import org.kurento.client.Continuation;
+import org.kurento.client.*;
-import org.kurento.client.ErrorEvent;
-import org.kurento.client.Filter;
-import org.kurento.client.IceCandidate;
-import org.kurento.client.MediaElement;
-import org.kurento.client.MediaPipeline;
-import org.kurento.client.MediaType;
-import org.kurento.client.SdpEndpoint;
 import org.kurento.client.internal.server.KurentoServerException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -498,6 +491,16 @@ public class Participant {
     room.sendMediaError(id, desc);
   }
 
+  public void sendMediaFlowInChange(MediaFlowInStateChangeEvent event) {
+    log.warn("PARTICIPANT {}: Media flow in: {}", name, event.getState());
+    room.sendMediaFlowInChange(name, event.getMediaType(),event.getState());
+  }
+
+  public void sendMediaFlowOutChange(MediaFlowOutStateChangeEvent event) {
+    log.warn("PARTICIPANT {}: Media flow out: {}", name, event.getState());
+    room.sendMediaFlowOutChange(name, event.getMediaType(),event.getState());
+  }
+
   private void releasePublisherEndpoint() {
     if (publisher != null && publisher.getEndpoint() != null) {
       this.streaming = false;
@@ -638,6 +641,7 @@ public class Participant {
 			System.out.println(msg2);
 			this.infoHandler.sendInfo(msg1);
 			this.infoHandler.sendInfo(msg2);
+            this.sendMediaFlowInChange(event);
 	  });
 	
 	  endpoint.getWebEndpoint().addMediaFlowOutStateChangeListener((event) -> { 
@@ -662,6 +666,7 @@ public class Participant {
 			System.out.println(msg2);
 			this.infoHandler.sendInfo(msg1);
 			this.infoHandler.sendInfo(msg2);
+			this.sendMediaFlowOutChange(event);
 		  });
 	
 	  endpoint.getWebEndpoint().addMediaSessionStartedListener((event) -> { 

openvidu-server/src/main/java/io/openvidu/server/core/internal/Room.java

@@ -24,12 +24,7 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import org.kurento.client.Continuation;
+import org.kurento.client.*;
-import org.kurento.client.ErrorEvent;
-import org.kurento.client.EventListener;
-import org.kurento.client.IceCandidate;
-import org.kurento.client.KurentoClient;
-import org.kurento.client.MediaPipeline;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -236,6 +231,16 @@ public class Room {
     this.roomHandler.onIceCandidate(name, participantId, endpointName, candidate);
   }
 
+  public void sendMediaFlowInChange(String participantId, MediaType mediaType,
+                                    MediaFlowState newState){
+    this.roomHandler.onMediaFlowInChange(name,participantId, mediaType, newState, participants.values());
+  }
+
+  public void sendMediaFlowOutChange(String participantId, MediaType mediaType,
+                                    MediaFlowState newState){
+    this.roomHandler.onMediaFlowOutChange(name,participantId, mediaType, newState, participants.values());
+  }
+
   public void sendMediaError(String participantId, String description) {
     this.roomHandler.onMediaElementError(name, participantId, description);
   }