From 33e29b26c33c58eb699066ec3c704985f6758f18 Mon Sep 17 00:00:00 2001 From: pabloFuente Date: Tue, 27 Feb 2018 14:37:39 +0100 Subject: [PATCH] 'videoElementDestroyed' event for Subscriber and Publisher. WebRtc stats for ElasTest. es5 to es6. webrtc-adapter to 6.1.1 --- openvidu-browser/package.json | 52 +-- openvidu-browser/ts/OpenVidu/Publisher.ts | 6 +- openvidu-browser/ts/OpenVidu/Session.ts | 2 +- openvidu-browser/ts/OpenVidu/Subscriber.ts | 6 +- .../ts/OpenViduInternal/OpenViduInternal.ts | 2 +- .../ts/OpenViduInternal/SessionInternal.ts | 8 +- .../ts/OpenViduInternal/Stream.ts | 37 +- .../ts/OpenViduInternal/WebRtcStats.ts | 321 ++++++++++++++++++ .../ts/OpenViduInternal/tsconfig.json | 2 +- 9 files changed, 394 insertions(+), 42 deletions(-) create mode 100644 openvidu-browser/ts/OpenViduInternal/WebRtcStats.ts diff --git a/openvidu-browser/package.json b/openvidu-browser/package.json index 3aacab1c..406c8a90 100644 --- a/openvidu-browser/package.json +++ b/openvidu-browser/package.json @@ -1,37 +1,37 @@ { - "author": "OpenVidu", + "author": "OpenVidu", "dependencies": { - "freeice": "2.2.0", - "hark": "1.1.6", - "inherits": "2.0.3", - "merge": "1.2.0", - "sdp-translator": "0.1.24", - "ua-parser-js": "0.7.17", - "uuid": "3.1.0", - "webrtc-adapter": "6.0.4", + "freeice": "2.2.0", + "hark": "1.1.6", + "inherits": "2.0.3", + "merge": "1.2.0", + "sdp-translator": "0.1.24", + "ua-parser-js": "0.7.17", + "uuid": "3.1.0", + "webrtc-adapter": "6.1.1", "wolfy87-eventemitter": "5.2.4" - }, - "description": "OpenVidu Browser", + }, + "description": "OpenVidu Browser", "devDependencies": { - "browserify": "15.1.0", - "tsify": "3.0.4", - "typescript": "2.6.2", + "browserify": "15.1.0", + "tsify": "3.0.4", + "typescript": "2.6.2", "uglify-js": "3.3.5" - }, - "license": "Apache-2.0", - "main": "lib/OpenVidu/index.js", - "name": "openvidu-browser", + }, + "license": "Apache-2.0", + "main": "lib/OpenVidu/index.js", + "name": "openvidu-browser", "repository": { - "type": "git", + "type": "git", "url": "git://github.com/OpenVidu/openvidu" - }, + }, "scripts": { - "browserify": "VERSION=${VERSION:-}; cd ts/OpenVidu && browserify Main.ts -p [ tsify ] --exclude kurento-browser-extensions --debug -o ../../static/js/openvidu-browser-$VERSION.js -v", - "browserify-prod": "VERSION=${VERSION:-}; cd ts/OpenVidu && browserify --debug Main.ts -p [ tsify ] --exclude kurento-browser-extensions | uglifyjs --source-map content=inline --output ../../static/js/openvidu-browser-$VERSION.min.js", - "prepublish": "cd ts/OpenViduInternal && tsc && cd ../OpenVidu && tsc && cd ../.. && tsc --declaration ts/OpenVidu/index.ts --outDir lib --sourceMap && tsc --declaration ts/OpenVidu/Main.ts --outDir lib --sourceMap", - "test": "echo \"Error: no test specified\" && exit 1", + "browserify": "VERSION=${VERSION:-}; cd ts/OpenVidu && browserify Main.ts -p [ tsify ] --exclude kurento-browser-extensions --debug -o ../../static/js/openvidu-browser-$VERSION.js -v", + "browserify-prod": "VERSION=${VERSION:-}; cd ts/OpenVidu && browserify --debug Main.ts -p [ tsify ] --exclude kurento-browser-extensions | uglifyjs --source-map content=inline --output ../../static/js/openvidu-browser-$VERSION.min.js", + "prepublish": "cd ts/OpenViduInternal && tsc && cd ../OpenVidu && tsc && cd ../.. && tsc --declaration ts/OpenVidu/index.ts --outDir lib --sourceMap && tsc --declaration ts/OpenVidu/Main.ts --outDir lib --sourceMap", + "test": "echo \"Error: no test specified\" && exit 1", "updatetsc": "cd ts/OpenViduInternal && tsc && cd ../OpenVidu && tsc && cd ../.. && tsc --declaration ts/OpenVidu/index.ts --outDir lib --sourceMap && tsc --declaration ts/OpenVidu/Main.ts --outDir lib --sourceMap" - }, - "types": "lib/OpenVidu/index.d.ts", + }, + "types": "lib/OpenVidu/index.d.ts", "version": "1.7.0" } \ No newline at end of file diff --git a/openvidu-browser/ts/OpenVidu/Publisher.ts b/openvidu-browser/ts/OpenVidu/Publisher.ts index 3c237d7f..d3180075 100644 --- a/openvidu-browser/ts/OpenVidu/Publisher.ts +++ b/openvidu-browser/ts/OpenVidu/Publisher.ts @@ -27,7 +27,11 @@ export class Publisher { // Listens to the deactivation of the default behaviour upon the deletion of a Stream object this.ee.addListener('stream-destroyed-default', event => { - event.stream.removeVideo(); + let s: Stream = event.stream; + s.addOnceEventListener('video-removed', () => { + this.ee.emitEvent('videoElementDestroyed'); + }); + s.removeVideo(); }); if (document.getElementById(parentId) != null) { diff --git a/openvidu-browser/ts/OpenVidu/Session.ts b/openvidu-browser/ts/OpenVidu/Session.ts index 85079894..ecf69f6d 100644 --- a/openvidu-browser/ts/OpenVidu/Session.ts +++ b/openvidu-browser/ts/OpenVidu/Session.ts @@ -99,7 +99,7 @@ export class Session { } } } - + private streamPublish(publisher: Publisher) { publisher.session = this; publisher.stream.publish(); diff --git a/openvidu-browser/ts/OpenVidu/Subscriber.ts b/openvidu-browser/ts/OpenVidu/Subscriber.ts index 5b485f8b..5743f46d 100644 --- a/openvidu-browser/ts/OpenVidu/Subscriber.ts +++ b/openvidu-browser/ts/OpenVidu/Subscriber.ts @@ -15,6 +15,11 @@ export class Subscriber { if (document.getElementById(parentId) != null) { this.element = document.getElementById(parentId)!!; } + + // Listens to deletion of the HTML video element of the Subscriber + this.stream.addEventListener('video-removed', () => { + this.ee.emitEvent('videoElementDestroyed'); + }); } on(eventName: string, callback) { @@ -33,7 +38,6 @@ export class Subscriber { }]); } else { this.stream.addOnceEventListener('video-element-created-by-stream', element => { - console.warn("Subscriber emitting videoElementCreated"); this.id = element.id; this.ee.emitEvent('videoElementCreated', [{ element: element diff --git a/openvidu-browser/ts/OpenViduInternal/OpenViduInternal.ts b/openvidu-browser/ts/OpenViduInternal/OpenViduInternal.ts index 350692fa..5a84aaf4 100644 --- a/openvidu-browser/ts/OpenViduInternal/OpenViduInternal.ts +++ b/openvidu-browser/ts/OpenViduInternal/OpenViduInternal.ts @@ -236,7 +236,7 @@ export class OpenViduInternal { if (this.session !== undefined && this.session instanceof SessionInternal) { return true; } else { - console.warn('Room instance not found'); + console.warn('Session instance not found'); return false; } } diff --git a/openvidu-browser/ts/OpenViduInternal/SessionInternal.ts b/openvidu-browser/ts/OpenViduInternal/SessionInternal.ts index 3b0d67eb..50c8672c 100644 --- a/openvidu-browser/ts/OpenViduInternal/SessionInternal.ts +++ b/openvidu-browser/ts/OpenViduInternal/SessionInternal.ts @@ -472,21 +472,21 @@ export class SessionInternal { onRoomClosed(msg) { - console.info("Room closed: " + JSON.stringify(msg)); + console.info("Session closed: " + JSON.stringify(msg)); let room = msg.room; if (room !== undefined) { this.ee.emitEvent('room-closed', [{ room: room }]); } else { - console.warn("Room undefined in on room closed", msg); + console.warn("Session undefined on session closed", msg); } } onLostConnection() { if (!this.connected) { - console.warn('Not connected to room: if you are not debugging, this is probably a certificate error'); + console.warn('Not connected to session: if you are not debugging, this is probably a certificate error'); if (window.confirm('If you are not debugging, this is probably a certificate error at \"' + this.openVidu.getOpenViduServerURL() + '\"\n\nClick OK to navigate and accept it')) { location.assign(this.openVidu.getOpenViduServerURL() + '/accept-certificate'); }; @@ -498,7 +498,7 @@ export class SessionInternal { if (room !== undefined) { this.ee.emitEvent('lost-connection', [{ room }]); } else { - console.warn('Room undefined when lost connection'); + console.warn('Session undefined when lost connection'); } } diff --git a/openvidu-browser/ts/OpenViduInternal/Stream.ts b/openvidu-browser/ts/OpenViduInternal/Stream.ts index e219f8e6..b4c86219 100644 --- a/openvidu-browser/ts/OpenViduInternal/Stream.ts +++ b/openvidu-browser/ts/OpenViduInternal/Stream.ts @@ -9,10 +9,12 @@ import { Connection } from './Connection'; import { SessionInternal } from './SessionInternal'; import { OpenViduInternal, Callback } from './OpenViduInternal'; import { OpenViduError, OpenViduErrorName } from './OpenViduError'; +import { WebRtcStats } from './WebRtcStats'; import EventEmitter = require('wolfy87-eventemitter'); import * as kurentoUtils from '../KurentoUtils/kurento-utils-js'; import * as adapter from 'webrtc-adapter'; + declare var navigator: any; declare var RTCSessionDescription: any; @@ -87,6 +89,8 @@ export class Stream { public isScreenRequestedReady: boolean = false; private isScreenRequested = false; + private webRtcStats: WebRtcStats; + constructor(private openVidu: OpenViduInternal, private local: boolean, private room: SessionInternal, options: any) { if (options !== 'screen-options') { if ('id' in options) { @@ -120,12 +124,14 @@ export class Stream { if (this.video) { if (typeof parentElement === "string") { document.getElementById(parentElement)!.removeChild(this.video); + this.ee.emitEvent('video-removed'); } else if (parentElement instanceof Element) { parentElement.removeChild(this.video); - } - else if (!parentElement) { + this.ee.emitEvent('video-removed'); + } else if (!parentElement) { if (document.getElementById(this.parentId)) { document.getElementById(this.parentId)!.removeChild(this.video); + this.ee.emitEvent('video-removed'); } } delete this.video; @@ -225,6 +231,10 @@ export class Stream { return this.wp; } + getRTCPeerConnection() { + return this.wp.peerConnection; + } + addEventListener(eventName: string, listener: any) { this.ee.addListener(eventName, listener); } @@ -324,10 +334,6 @@ export class Stream { return this.connection; } - getRTCPeerConnection() { - return this.getWebRtcPeer().peerConnection; - } - requestCameraAccess(callback: Callback) { this.connection.addStream(this); @@ -433,7 +439,7 @@ export class Stream { doLoopback: this.displayMyRemote() || false, audioActive: this.outboundOptions.sendAudio, videoActive: this.outboundOptions.sendVideo, - typeOfVideo: ((this.outboundOptions.sendVideo) ? ((this.isScreenRequested) ? 'SCREEN' :'CAMERA') : '') + typeOfVideo: ((this.outboundOptions.sendVideo) ? ((this.isScreenRequested) ? 'SCREEN' : 'CAMERA') : '') }, (error, response) => { if (error) { console.error("Error on publishVideo: " + JSON.stringify(error)); @@ -614,6 +620,9 @@ export class Stream { stream: this }]); } + + this.initWebRtcStats(); + }, error => { console.error(this.streamId + ": Error setting SDP to the peer connection: " + JSON.stringify(error)); @@ -668,6 +677,8 @@ export class Stream { this.speechEvent.stop(); } + this.stopWebRtcStats(); + console.info((this.local ? "Local " : "Remote ") + "'Stream' with id [" + this.streamId + "]' has been succesfully disposed"); } @@ -675,4 +686,16 @@ export class Stream { this.outboundOptions = options; this.streamId = "SCREEN"; } + + private initWebRtcStats(): void { + this.webRtcStats = new WebRtcStats(this); + this.webRtcStats.initWebRtcStats(); + } + + private stopWebRtcStats() { + if (this.webRtcStats.isEnabled()) { + this.webRtcStats.stopWebRtcStats(); + } + } + } diff --git a/openvidu-browser/ts/OpenViduInternal/WebRtcStats.ts b/openvidu-browser/ts/OpenViduInternal/WebRtcStats.ts new file mode 100644 index 00000000..e3744241 --- /dev/null +++ b/openvidu-browser/ts/OpenViduInternal/WebRtcStats.ts @@ -0,0 +1,321 @@ +import { Stream } from './Stream'; +import * as adapter from 'webrtc-adapter'; + +export class WebRtcStats { + + private webRtcStatsEnabled: boolean = false; + private webRtcStatsIntervalId: number; + private statsInterval: number = 1; + private stats: any = { + "inbound": { + "audio": { + "bytesReceived": 0, + "packetsReceived": 0, + "packetsLost": 0 + }, + "video": { + "bytesReceived": 0, + "packetsReceived": 0, + "packetsLost": 0, + "framesDecoded": 0, + "nackCount": 0 + } + }, + "outbound": { + "audio": { + "bytesSent": 0, + "packetsSent": 0, + }, + "video": { + "bytesSent": 0, + "packetsSent": 0, + "framesEncoded": 0, + "nackCount": 0 + } + } + } + + constructor(private stream: Stream) { } + + public isEnabled(): boolean { + return this.webRtcStatsEnabled; + } + + public initWebRtcStats(): void { + + let elastestInstrumentation = localStorage.getItem('elastest-instrumentation'); + + if (elastestInstrumentation) { + // ElasTest instrumentation object found in local storage + + console.warn("WebRtc stats enabled for stream " + this.stream.streamId + " of connection " + this.stream.connection.connectionId); + + this.webRtcStatsEnabled = true; + + let instrumentation = JSON.parse(elastestInstrumentation); + this.statsInterval = instrumentation.webrtc.interval; // Interval in seconds + + console.warn("localStorage item: " + JSON.stringify(instrumentation)); + + this.webRtcStatsIntervalId = setInterval(() => { + this.sendStatsToHttpEndpoint(instrumentation); + }, this.statsInterval * 1000); + + return; + } + + console.debug("WebRtc stats not enabled"); + } + + public stopWebRtcStats() { + if (this.webRtcStatsEnabled) { + clearInterval(this.webRtcStatsIntervalId); + console.warn("WebRtc stats stopped for disposed stream " + this.stream.streamId + " of connection " + this.stream.connection.connectionId); + } + } + + private sendStatsToHttpEndpoint(instrumentation): void { + + let sendPost = (json) => { + let http: XMLHttpRequest = new XMLHttpRequest(); + let url: string = instrumentation.webrtc.httpEndpoint; + http.open("POST", url, true); + + http.setRequestHeader("Content-type", "application/json"); + + http.onreadystatechange = () => { // Call a function when the state changes. + if (http.readyState == 4 && http.status == 200) { + console.log("WebRtc stats succesfully sent to " + url + " for stream " + this.stream.streamId + " of connection " + this.stream.connection.connectionId); + } + } + http.send(JSON.stringify(json)); + } + + let f = (stats) => { + + if (adapter.browserDetails.browser === 'firefox') { + stats.forEach((stat) => { + + let json = {}; + + if ((stat.type === 'inbound-rtp') && + ( + // Avoid firefox empty outbound-rtp statistics + stat.nackCount != null && + stat.isRemote === false && + stat.id.startsWith('inbound') && + stat.remoteId.startsWith('inbound') + )) { + + let metricId = 'webrtc_inbound_' + stat.mediaType + '_' + stat.ssrc; + let jitter = stat.jitter * 1000; + + let metrics = { + "bytesReceived": (stat.bytesReceived - this.stats.inbound[stat.mediaType].bytesReceived) / this.statsInterval, + "jitter": jitter, + "packetsReceived": (stat.packetsReceived - this.stats.inbound[stat.mediaType].packetsReceived) / this.statsInterval, + "packetsLost": (stat.packetsLost - this.stats.inbound[stat.mediaType].packetsLost) / this.statsInterval + }; + let units = { + "bytesReceived": "bytes", + "jitter": "ms", + "packetsReceived": "packets", + "packetsLost": "packets" + }; + if (stat.mediaType === 'video') { + metrics['framesDecoded'] = (stat.framesDecoded - this.stats.inbound.video.framesDecoded) / this.statsInterval; + metrics['nackCount'] = (stat.nackCount - this.stats.inbound.video.nackCount) / this.statsInterval; + units['framesDecoded'] = "frames"; + units['nackCount'] = "packets"; + + this.stats.inbound.video.framesDecoded = stat.framesDecoded; + this.stats.inbound.video.nackCount = stat.nackCount; + } + + this.stats.inbound[stat.mediaType].bytesReceived = stat.bytesReceived; + this.stats.inbound[stat.mediaType].packetsReceived = stat.packetsReceived; + this.stats.inbound[stat.mediaType].packetsLost = stat.packetsLost; + + json = { + "@timestamp": new Date(stat.timestamp).toISOString(), + "exec": instrumentation.exec, + "component": instrumentation.component, + "type": metricId, + "stream_type": "composed_metric", + "units": units + } + json[metricId] = metrics; + + sendPost(JSON.stringify(json)); + + } else if ((stat.type === 'outbound-rtp') && + ( + // Avoid firefox empty inbound-rtp statistics + stat.isRemote === false && + stat.id.toLowerCase().includes('outbound') + )) { + + let metricId = 'webrtc_outbound_' + stat.mediaType + '_' + stat.ssrc; + + let metrics = { + "bytesSent": (stat.bytesSent - this.stats.outbound[stat.mediaType].bytesSent) / this.statsInterval, + "packetsSent": (stat.packetsSent - this.stats.outbound[stat.mediaType].packetsSent) / this.statsInterval + }; + let units = { + "bytesSent": "bytes", + "packetsSent": "packets" + }; + if (stat.mediaType === 'video') { + metrics['framesEncoded'] = (stat.framesEncoded - this.stats.outbound.video.framesEncoded) / this.statsInterval; + units['framesEncoded'] = "frames"; + + this.stats.outbound.video.framesEncoded = stat.framesEncoded; + } + + this.stats.outbound[stat.mediaType].bytesSent = stat.bytesSent; + this.stats.outbound[stat.mediaType].packetsSent = stat.packetsSent; + + json = { + "@timestamp": new Date(stat.timestamp).toISOString(), + "exec": instrumentation.exec, + "component": instrumentation.component, + "type": metricId, + "stream_type": "composed_metric", + "units": units + } + json[metricId] = metrics; + + sendPost(JSON.stringify(json)); + } + }); + } else if (adapter.browserDetails.browser === 'chrome') { + for (let key of Object.keys(stats)) { + let stat = stats[key]; + if (stat.type === 'ssrc') { + + let json = {}; + + if ('bytesReceived' in stat && ( + (stat.mediaType === 'audio' && 'audioOutputLevel' in stat) || + (stat.mediaType === 'video' && 'qpSum' in stat) + )) { + // inbound-rtp + let metricId = 'webrtc_inbound_' + stat.mediaType + '_' + stat.ssrc; + + let metrics = { + "bytesReceived": (stat.bytesReceived - this.stats.inbound[stat.mediaType].bytesReceived) / this.statsInterval, + "jitter": stat.googJitterBufferMs, + "packetsReceived": (stat.packetsReceived - this.stats.inbound[stat.mediaType].packetsReceived) / this.statsInterval, + "packetsLost": (stat.packetsLost - this.stats.inbound[stat.mediaType].packetsLost) / this.statsInterval + }; + let units = { + "bytesReceived": "bytes", + "jitter": "ms", + "packetsReceived": "packets", + "packetsLost": "packets" + }; + if (stat.mediaType === 'video') { + metrics['framesDecoded'] = (stat.framesDecoded - this.stats.inbound.video.framesDecoded) / this.statsInterval; + metrics['nackCount'] = (stat.googNacksSent - this.stats.inbound.video.nackCount) / this.statsInterval; + units['framesDecoded'] = "frames"; + units['nackCount'] = "packets"; + + this.stats.inbound.video.framesDecoded = stat.framesDecoded; + this.stats.inbound.video.nackCount = stat.googNacksSent; + } + + this.stats.inbound[stat.mediaType].bytesReceived = stat.bytesReceived; + this.stats.inbound[stat.mediaType].packetsReceived = stat.packetsReceived; + this.stats.inbound[stat.mediaType].packetsLost = stat.packetsLost; + + json = { + "@timestamp": new Date(stat.timestamp).toISOString(), + "exec": instrumentation.exec, + "component": instrumentation.component, + "type": metricId, + "stream_type": "composed_metric", + "units": units + } + json[metricId] = metrics; + + sendPost(JSON.stringify(json)); + } else if ('bytesSent' in stat) { + // outbound-rtp + let metricId = 'webrtc_outbound_' + stat.mediaType + '_' + stat.ssrc; + + let metrics = { + "bytesSent": (stat.bytesSent - this.stats.outbound[stat.mediaType].bytesSent) / this.statsInterval, + "packetsSent": (stat.packetsSent - this.stats.outbound[stat.mediaType].packetsSent) / this.statsInterval + }; + let units = { + "bytesSent": "bytes", + "packetsSent": "packets" + }; + if (stat.mediaType === 'video') { + metrics['framesEncoded'] = (stat.framesEncoded - this.stats.outbound.video.framesEncoded) / this.statsInterval; + units['framesEncoded'] = "frames"; + + this.stats.outbound.video.framesEncoded = stat.framesEncoded; + } + + this.stats.outbound[stat.mediaType].bytesSent = stat.bytesSent; + this.stats.outbound[stat.mediaType].packetsSent = stat.packetsSent; + + json = { + "@timestamp": new Date(stat.timestamp).toISOString(), + "exec": instrumentation.exec, + "component": instrumentation.component, + "type": metricId, + "stream_type": "composed_metric", + "units": units + } + json[metricId] = metrics; + + sendPost(JSON.stringify(json)); + } + } + } + } + }; + + this.getStatsAgnostic(this.stream.getRTCPeerConnection(), null, f, (error) => { console.log(error) }); + } + + private standardizeReport(response) { + if (adapter.browserDetails.browser === 'firefox') { + return response; + } + + var standardReport = {}; + response.result().forEach(function (report) { + var standardStats = { + id: report.id, + timestamp: report.timestamp, + type: report.type + }; + report.names().forEach(function (name) { + standardStats[name] = report.stat(name); + }); + standardReport[standardStats.id] = standardStats; + }); + + return standardReport; + } + + private getStatsAgnostic(pc, selector, successCb, failureCb) { + if (adapter.browserDetails.browser === 'firefox') { + // getStats takes args in different order in Chrome and Firefox + return pc.getStats(selector, (response) => { + var report = this.standardizeReport(response); + successCb(report); + }, failureCb); + } else if (adapter.browserDetails.browser === 'chrome') { + // In Chrome, the first two arguments are reversed + return pc.getStats((response) => { + var report = this.standardizeReport(response); + successCb(report); + }, selector, failureCb); + } + } + +} \ No newline at end of file diff --git a/openvidu-browser/ts/OpenViduInternal/tsconfig.json b/openvidu-browser/ts/OpenViduInternal/tsconfig.json index beaf47be..272c1d31 100644 --- a/openvidu-browser/ts/OpenViduInternal/tsconfig.json +++ b/openvidu-browser/ts/OpenViduInternal/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { "allowJs": true, - "target": "es5", + "target": "es6", "module": "commonjs", //"noImplicitAny": true, "noImplicitThis": true,