From 0b7fefd90688a2d65dd591a8288ac55744b9650c Mon Sep 17 00:00:00 2001 From: Carlos Santos <4a.santos@gmail.com> Date: Fri, 18 Nov 2022 11:54:38 +0100 Subject: [PATCH] openvidu-components: Added reconnection when STT disconnected --- .../captions/captions.component.css | 16 +++ .../captions/captions.component.html | 6 + .../components/captions/captions.component.ts | 54 +++++---- .../components/session/session.component.ts | 49 ++++---- .../settings/captions/captions.component.html | 2 +- .../settings/captions/captions.component.ts | 21 +++- .../openvidu-angular/src/lib/lang/cn.json | 3 +- .../openvidu-angular/src/lib/lang/de.json | 3 +- .../openvidu-angular/src/lib/lang/en.json | 3 +- .../openvidu-angular/src/lib/lang/es.json | 3 +- .../openvidu-angular/src/lib/lang/fr.json | 3 +- .../openvidu-angular/src/lib/lang/hi.json | 3 +- .../openvidu-angular/src/lib/lang/it.json | 3 +- .../openvidu-angular/src/lib/lang/ja.json | 3 +- .../openvidu-angular/src/lib/lang/nl.json | 3 +- .../openvidu-angular/src/lib/lang/pt.json | 3 +- .../lib/services/caption/caption.service.ts | 6 +- .../lib/services/openvidu/openvidu.service.ts | 106 +++++++++++++++--- 18 files changed, 209 insertions(+), 81 deletions(-) diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.css b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.css index 6e302e3b..c5ff384e 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.css +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.css @@ -26,6 +26,22 @@ padding: 0px 10vw 0px; } +.error-container { + display: grid; + text-align: center; + color: var(--ov-text-color); + font-size: 18px; +} + +mat-spinner { + position: relative; + top: 35%; + bottom: 0; + left: 0; + right: 0; + margin: auto; +} + /* * Screen XL diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.html b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.html index b550a523..acc0316e 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.html +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.html @@ -10,6 +10,7 @@
+ + {{'ERRORS.SST_CONNECTION' | translate}} + +
diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.ts b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.ts index 257c47db..816f21aa 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.ts +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.ts @@ -42,6 +42,7 @@ export class CaptionsComponent implements OnInit { captionEvents: CaptionModel[] = []; session: Session; + isSttReady: boolean = true; private deleteFirstTimeout: NodeJS.Timeout; private deleteAllTimeout: NodeJS.Timeout; @@ -52,6 +53,8 @@ export class CaptionsComponent implements OnInit { private captionLangSelected: { name: string; ISO: string }; private screenSizeSub: Subscription; private panelTogglingSubscription: Subscription; + private sttStatusSubscription: Subscription; + constructor( private panelService: PanelService, @@ -62,14 +65,12 @@ export class CaptionsComponent implements OnInit { ) {} async ngOnInit(): Promise { + this.subscribeToSTTStatus(); this.captionService.setCaptionsEnabled(true); this.captionLangSelected = this.captionService.getLangSelected(); this.session = this.openviduService.getWebcamSession(); - for (const p of this.participantService.getRemoteParticipants()) { - const stream = p.getCameraConnection().streamManager.stream; - await this.session.subscribeToSpeechToText(stream, this.captionLangSelected.ISO); - } + await this.openviduService.subscribeRemotesToSTT(this.captionLangSelected.ISO); this.subscribeToCaptionLanguage(); this.subscribeToPanelToggling(); @@ -77,16 +78,14 @@ export class CaptionsComponent implements OnInit { } async ngOnDestroy() { + await this.openviduService.unsubscribeRemotesFromSTT(); this.captionService.setCaptionsEnabled(false); if (this.screenSizeSub) this.screenSizeSub.unsubscribe(); if (this.panelTogglingSubscription) this.panelTogglingSubscription.unsubscribe(); + if(this.sttStatusSubscription) this.sttStatusSubscription.unsubscribe(); this.session.off('speechToTextMessage'); this.captionEvents = []; - for (const p of this.participantService.getRemoteParticipants()) { - const stream = p.getCameraConnection().streamManager.stream; - await this.session.unsubscribeFromSpeechToText(stream); - } } onSettingsCliked() { @@ -95,22 +94,24 @@ export class CaptionsComponent implements OnInit { private subscribeToTranscription() { this.session.on('speechToTextMessage', (event: SpeechToTextEvent) => { - clearInterval(this.deleteAllTimeout); - const { connectionId, data } = event.connection; - const nickname: string = this.participantService.getNicknameFromConnectionData(data); - const color = this.participantService.getRemoteParticipantByConnectionId(connectionId)?.colorProfile || ''; + if(!!event.text) { + clearInterval(this.deleteAllTimeout); + const { connectionId, data } = event.connection; + const nickname: string = this.participantService.getNicknameFromConnectionData(data); + const color = this.participantService.getRemoteParticipantByConnectionId(connectionId)?.colorProfile || ''; - const caption: CaptionModel = { - connectionId, - nickname, - color, - text: event.text, - type: event.reason - }; - this.updateCaption(caption); - // Delete all events when there are no more events for a period of time - this.deleteAllEventsAfterDelay(this.DELETE_TIMEOUT); - this.cd.markForCheck(); + const caption: CaptionModel = { + connectionId, + nickname, + color, + text: event.text, + type: event.reason + }; + this.updateCaption(caption); + // Delete all events when there are no more events for a period of time + this.deleteAllEventsAfterDelay(this.DELETE_TIMEOUT); + this.cd.markForCheck(); + } }); } private updateCaption(caption: CaptionModel): void { @@ -187,6 +188,13 @@ export class CaptionsComponent implements OnInit { }, timeout); } + private subscribeToSTTStatus() { + this.sttStatusSubscription = this.openviduService.isSttReadyObs.subscribe((ready: boolean) => { + this.isSttReady = ready; + this.cd.markForCheck(); + }); + } + private subscribeToCaptionLanguage() { this.captionLanguageSubscription = this.captionService.captionLangObs.subscribe((lang) => { this.captionLangSelected = lang; diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/session/session.component.ts b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/session/session.component.ts index 03e169d3..d89afa3a 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/session/session.component.ts +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/session/session.component.ts @@ -16,6 +16,7 @@ import { import { ConnectionEvent, ExceptionEvent, + ExceptionEventName, RecordingEvent, Session, SessionDisconnectedEvent, @@ -84,6 +85,7 @@ export class SessionComponent implements OnInit, OnDestroy { protected layoutWidthSubscription: Subscription; protected updateLayoutInterval: NodeJS.Timer; + // private sttReconnectionInterval: NodeJS.Timer; private captionLanguageSubscription: Subscription; protected log: ILogger; @@ -277,8 +279,15 @@ export class SessionComponent implements OnInit, OnDestroy { } private subscribeToOpenViduException() { - this.session.on('exception', (event: ExceptionEvent) => { - this.log.e(event.name, event.message); + this.session.on('exception', async (event: ExceptionEvent) => { + if (event.name === ExceptionEventName.SPEECH_TO_TEXT_DISCONNECTED) { + this.log.w(event.name, event.message); + this.openviduService.setSTTReady(false); + // Try to re-subscribe to STT + await this.openviduService.subscribeRemotesToSTT(this.captionService.getLangSelected().ISO); + } else { + this.log.e(event.name, event.message); + } }); } @@ -319,18 +328,23 @@ export class SessionComponent implements OnInit, OnDestroy { const data = event.stream?.connection?.data; const isCameraType: boolean = this.participantService.getTypeConnectionData(data) === VideoType.CAMERA; const isRemoteConnection: boolean = !this.openviduService.isMyOwnConnection(connectionId); + const lang = this.captionService.getLangSelected().ISO; if (isRemoteConnection) { const subscriber: Subscriber = this.session.subscribe(event.stream, undefined); this.participantService.addRemoteConnection(connectionId, data, subscriber); // this.oVSessionService.sendNicknameSignal(event.stream.connection); - if (this.captionService.areCaptionsEnabled() && isCameraType) { - // Only subscribe to STT when stream is CAMERA type and it is a remote stream + if (this.openviduService.isSttReady() && this.captionService.areCaptionsEnabled() && isCameraType) { + // Only subscribe to STT when is ready and stream is CAMERA type and it is a remote stream try { - await this.session.subscribeToSpeechToText(event.stream, this.captionService.getLangSelected().ISO); + await this.openviduService.subscribeStreamToStt(event.stream, lang); } catch (error) { this.log.e('Error subscribing from STT: ', error); + // I assume the only reason of an STT error is a STT crash. + // It must be subscribed to all remotes again + // await this.openviduService.unsubscribeRemotesFromSTT(); + await this.openviduService.subscribeRemotesToSTT(lang); } } } @@ -345,13 +359,11 @@ export class SessionComponent implements OnInit, OnDestroy { const isCameraType: boolean = this.participantService.getTypeConnectionData(data) === VideoType.CAMERA; this.participantService.removeConnectionByConnectionId(connectionId); - if (isRemoteConnection) { - if (this.captionService.areCaptionsEnabled() && isCameraType) { - try { - await this.session.unsubscribeFromSpeechToText(event.stream); - } catch (error) { - this.log.e('Error unsubscribing from STT: ', error); - } + if (this.openviduService.isSttReady() && this.captionService.areCaptionsEnabled() && isRemoteConnection && isCameraType) { + try { + await this.session.unsubscribeFromSpeechToText(event.stream); + } catch (error) { + this.log.e('Error unsubscribing from STT: ', error); } } }); @@ -362,17 +374,8 @@ export class SessionComponent implements OnInit, OnDestroy { if (this.captionService.areCaptionsEnabled()) { // Unsubscribe all streams from speech to text and re-subscribe with new language this.log.d('Re-subscribe from STT because of language changed to ', lang.ISO); - for (const participant of this.participantService.getRemoteParticipants()) { - const streamManager = participant.getCameraConnection()?.streamManager; - if (!!streamManager?.stream) { - try { - await this.session.unsubscribeFromSpeechToText(streamManager.stream); - await this.session.subscribeToSpeechToText(streamManager.stream, lang.ISO); - } catch (error) { - this.log.e('Error re-subscribing to STT: ', error); - } - } - } + await this.openviduService.unsubscribeRemotesFromSTT(); + await this.openviduService.subscribeRemotesToSTT(lang.ISO); } }); } diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/settings/captions/captions.component.html b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/settings/captions/captions.component.html index 78147dde..3f4f33b3 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/settings/captions/captions.component.html +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/settings/captions/captions.component.html @@ -15,7 +15,7 @@
{{ 'PANEL.SETTINGS.LANGUAGE' | translate }}
- diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/settings/captions/captions.component.ts b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/settings/captions/captions.component.ts index 4da6ad8a..5e19b12f 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/settings/captions/captions.component.ts +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/settings/captions/captions.component.ts @@ -13,25 +13,30 @@ import { OpenViduService } from '../../../services/openvidu/openvidu.service'; styleUrls: ['./captions.component.css'] }) export class CaptionsSettingComponent implements OnInit, OnDestroy { + isSttReady: boolean = true; captionsEnabled: boolean; languagesAvailable: { name: string; ISO: string }[] = []; - captionsSubscription: Subscription; langSelected: string; isOpenViduPro: boolean = false; + private captionsStatusSubs: Subscription; + private sttStatusSubs: Subscription; + constructor(private layoutService: LayoutService, private captionService: CaptionService, private openviduService: OpenViduService) {} ngOnInit(): void { this.isOpenViduPro = this.openviduService.isOpenViduPro(); if (this.isOpenViduPro) { - this.subscribeToCaptions(); + this.subscribeToSttStatus(); + this.subscribeToCaptionsStatus(); this.langSelected = this.captionService.getLangSelected().name; this.languagesAvailable = this.captionService.getCaptionLanguages(); } } ngOnDestroy() { - if (this.captionsSubscription) this.captionsSubscription.unsubscribe(); + if (this.captionsStatusSubs) this.captionsStatusSubs.unsubscribe(); + if (this.sttStatusSubs) this.sttStatusSubs.unsubscribe(); } onLangSelected(lang: { name: string; ISO: string }) { @@ -43,8 +48,14 @@ export class CaptionsSettingComponent implements OnInit, OnDestroy { this.layoutService.toggleCaptions(); } - private subscribeToCaptions() { - this.captionsSubscription = this.layoutService.captionsTogglingObs.subscribe((value: boolean) => { + private subscribeToSttStatus(){ + this.sttStatusSubs = this.openviduService.isSttReadyObs.subscribe((ready: boolean) => { + this.isSttReady = ready; + }); + } + + private subscribeToCaptionsStatus() { + this.captionsStatusSubs = this.layoutService.captionsTogglingObs.subscribe((value: boolean) => { this.captionsEnabled = value; // this.cd.markForCheck(); }); diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/cn.json b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/cn.json index 1393cfb4..3a101f56 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/cn.json +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/cn.json @@ -112,6 +112,7 @@ "SCREEN_SHARING": "分享屏幕出错", "SCREEN_SUPPORT": "您的浏览器不支持屏幕共享", "MEDIA_ACCESS": "不允许访问媒体设备", - "DEVICE_NOT_FOUND": "没有找到视频或音频设备 请至少连接一个" + "DEVICE_NOT_FOUND": "没有找到视频或音频设备 请至少连接一个", + "SST_CONNECTION": "连接丢失。正在重新连接到语音到文本服务" } } diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/de.json b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/de.json index 93857057..d2426a32 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/de.json +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/de.json @@ -112,6 +112,7 @@ "SCREEN_SHARING": "Fehler beim Teilen des Bildschirms", "SCREEN_SUPPORT": "Ihr Browser unterstützt keine Bildschirmfreigabe", "MEDIA_ACCESS": "Der Zugriff auf Mediengeräte war nicht erlaubt.", - "DEVICE_NOT_FOUND": "Es wurden keine Video- oder Audiogeräte gefunden. Bitte schließen Sie mindestens eines an." + "DEVICE_NOT_FOUND": "Es wurden keine Video- oder Audiogeräte gefunden. Bitte schließen Sie mindestens eines an.", + "SST_CONNECTION": "Verbindung verloren. Wiederverbindung zum Sprach zu Text Service..." } } diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/en.json b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/en.json index 84ff19fe..2101b698 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/en.json +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/en.json @@ -113,6 +113,7 @@ "SCREEN_SHARING": "Error sharing screen", "SCREEN_SUPPORT": "Your browser does not support screen sharing", "MEDIA_ACCESS": "Access to media devices was not allowed.", - "DEVICE_NOT_FOUND": "No video or audio devices have been found. Please, connect at least one." + "DEVICE_NOT_FOUND": "No video or audio devices have been found. Please, connect at least one.", + "SST_CONNECTION": "Connection lost. Reconnecting to the speech to text service ..." } } diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/es.json b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/es.json index 56575bc1..afb24780 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/es.json +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/es.json @@ -112,6 +112,7 @@ "SCREEN_SHARING": "Hubo un error compartiendo pantalla", "SCREEN_SUPPORT": "Tu navegador no soporta la pantalla compartida", "MEDIA_ACCESS": "No se ha podido acceder a tus dispositivos", - "DEVICE_NOT_FOUND": "No se han encontrado dispositivos de audio o video. Por favor, conecta al menos uno." + "DEVICE_NOT_FOUND": "No se han encontrado dispositivos de audio o video. Por favor, conecta al menos uno.", + "SST_CONNECTION": "Conexión perdida. Reconectando al servicio de transcripción a texto..." } } diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/fr.json b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/fr.json index d263fac2..bed7585c 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/fr.json +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/fr.json @@ -112,6 +112,7 @@ "SCREEN_SHARING": "Erreur de partage d'écran", "SCREEN_SUPPORT": "Votre navigateur ne prend pas en charge le partage d'écran", "MEDIA_ACCESS": "L'accès aux périphériques médias n'a pas été autorisé", - "DEVICE_NOT_FOUND": "Aucun périphérique vidéo ou audio n'a été trouvé. Veuillez en connecter au moins un." + "DEVICE_NOT_FOUND": "Aucun périphérique vidéo ou audio n'a été trouvé. Veuillez en connecter au moins un.", + "SST_CONNECTION": "Connexion perdue. Reconnexion au service de reconnaissance vocale..." } } diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/hi.json b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/hi.json index 79620a8f..67eb80cf 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/hi.json +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/hi.json @@ -112,6 +112,7 @@ "SCREEN_SHARING": "स्क्रीन साझा करने में त्रुटि", "SCREEN_SUPPORT": "आपका ब्राउज़र स्क्रीन साझाकरण का समर्थन नहीं करता", "MEDIA_ACCESS": "मीडिया उपकरणों तक पहुंच की अनुमति नहीं थी।", - "DEVICE_NOT_FOUND": "कोई वीडियो या ऑडियो डिवाइस नहीं मिला। कृपया, कम से कम एक कनेक्ट करें।" + "DEVICE_NOT_FOUND": "कोई वीडियो या ऑडियो डिवाइस नहीं मिला। कृपया, कम से कम एक कनेक्ट करें।", + "SST_CONNECTION": "खोया तार। लेख सेवा से लिप्यंतरण से पुन: कनेक्ट हो रहा है..." } } diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/it.json b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/it.json index 6e32c2fc..6176ae4f 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/it.json +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/it.json @@ -112,6 +112,7 @@ "SCREEN_SHARING": "Errore nella condivisione dello schermo", "SCREEN_SUPPORT": "Il browser non supporta la condivisione dello schermo", "MEDIA_ACCESS": "L'accesso ai dispositivi multimediali non è stato consentito", - "DEVICE_NOT_FOUND": "Non sono stati trovati dispositivi video o audio. Si prega di collegarne almeno uno" + "DEVICE_NOT_FOUND": "Non sono stati trovati dispositivi video o audio. Si prega di collegarne almeno uno", + "SST_CONNECTION": "Connessione persa. Riconnessione al servizio di conversione testo da audio in corso..." } } diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/ja.json b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/ja.json index 06cd5816..97cf7be7 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/ja.json +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/ja.json @@ -112,6 +112,7 @@ "SCREEN_SHARING": "画面共有にエラーが発生しました", "SCREEN_SUPPORT": "お使いのブラウザは画面共有に対応していません", "MEDIA_ACCESS": "メディアデバイスへのアクセスが許可されませんでした", - "DEVICE_NOT_FOUND": "ビデオまたはオーディオデバイスが見つかりませんでした 最低1台は接続してください" + "DEVICE_NOT_FOUND": "ビデオまたはオーディオデバイスが見つかりませんでした 最低1台は接続してください", + "SST_CONNECTION": "接続が失われました。音声からテキストへの変換サービスに再接続しています" } } diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/nl.json b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/nl.json index b7970b11..8d6f9476 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/nl.json +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/nl.json @@ -112,6 +112,7 @@ "SCREEN_SHARING": "Fout bij het delen van het scherm", "SCREEN_SUPPORT": "Uw browser ondersteunt het delen van schermen niet", "MEDIA_ACCESS": "Toegang tot media-apparaten was niet toegestaan.", - "DEVICE_NOT_FOUND": "Er zijn geen video- of audioapparaten gevonden. Sluit er alstublieft ten minste één aan." + "DEVICE_NOT_FOUND": "Er zijn geen video- of audioapparaten gevonden. Sluit er alstublieft ten minste één aan.", + "SST_CONNECTION": "Verbinding verbroken. Opnieuw verbinden met de spraak-naar-tekstservice..." } } diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/pt.json b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/pt.json index 881c3271..24934425 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/pt.json +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/lang/pt.json @@ -112,6 +112,7 @@ "SCREEN_SHARING": "ecrã_partilha de erros", "SCREEN_SUPPORT": "O seu browser não suporta a partilha de ecrãs", "MEDIA_ACCESS": "Não foi permitido o acesso a dispositivos de media", - "DEVICE_NOT_FOUND": "Nenhum dispositivo de vídeo ou áudio foi encontrado. Por favor, ligue pelo menos um" + "DEVICE_NOT_FOUND": "Nenhum dispositivo de vídeo ou áudio foi encontrado. Por favor, ligue pelo menos um", + "SST_CONNECTION": "Conexão perdida. Reconectando ao serviço de texto de voz..." } } diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/services/caption/caption.service.ts b/openvidu-components-angular/projects/openvidu-angular/src/lib/services/caption/caption.service.ts index 9f5882f0..af721baf 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/services/caption/caption.service.ts +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/services/caption/caption.service.ts @@ -22,7 +22,7 @@ export class CaptionService { ]; captionLangSelected: { name: string; ISO: string } = { name: 'English', ISO: 'en-US' }; captionLangObs: Observable<{ name: string; ISO: string }>; - private _captionLangObs: Subject<{ name: string; ISO: string }> = new Subject(); + private _captionLang: Subject<{ name: string; ISO: string }> = new Subject(); private captionsEnabled: boolean = false; constructor(private storageService: StorageService) { @@ -33,7 +33,7 @@ export class CaptionService { } else { this.captionLangSelected = this.langs[0]; } - this.captionLangObs = this._captionLangObs.asObservable(); + this.captionLangObs = this._captionLang.asObservable(); } setCaptionsEnabled(value: boolean) { @@ -49,7 +49,7 @@ export class CaptionService { if (!!newLang && newLang.ISO !== this.captionLangSelected.ISO) { this.captionLangSelected = newLang; this.storageService.setCaptionLang(lang); - this._captionLangObs.next(this.captionLangSelected); + this._captionLang.next(this.captionLangSelected); } } diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/services/openvidu/openvidu.service.ts b/openvidu-components-angular/projects/openvidu-angular/src/lib/services/openvidu/openvidu.service.ts index 155b217c..223ae14f 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/services/openvidu/openvidu.service.ts +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/services/openvidu/openvidu.service.ts @@ -7,11 +7,13 @@ import { Publisher, PublisherProperties, Session, - SignalOptions + SignalOptions, + Stream } from 'openvidu-browser'; import { LoggerService } from '../logger/logger.service'; +import { BehaviorSubject, Observable } from 'rxjs'; import { CameraType } from '../../models/device.model'; import { ILogger } from '../../models/logger.model'; import { OpenViduEdition } from '../../models/openvidu.model'; @@ -26,6 +28,7 @@ import { PlatformService } from '../platform/platform.service'; providedIn: 'root' }) export class OpenViduService { + isSttReadyObs: Observable; private ovEdition: OpenViduEdition; private webcamToken = ''; private screenToken = ''; @@ -35,6 +38,9 @@ export class OpenViduService { protected screenSession: Session; protected videoSource = undefined; protected audioSource = undefined; + private STT_TIMEOUT_MS = 2 * 1000; + private sttReconnectionTimeout: NodeJS.Timeout; + private _isSttReady: BehaviorSubject = new BehaviorSubject(true); protected log: ILogger; /** @@ -48,6 +54,7 @@ export class OpenViduService { protected deviceService: DeviceService ) { this.log = this.loggerSrv.get('OpenViduService'); + this.isSttReadyObs = this._isSttReady.asObservable(); } /** @@ -109,7 +116,7 @@ export class OpenViduService { /** * @internal */ - isOpenViduPro(): boolean { + isOpenViduPro(): boolean { return this.ovEdition === OpenViduEdition.PRO; } @@ -170,6 +177,25 @@ export class OpenViduService { return !!this.screenSession.capabilities; } + /** + * @internal + * Whether the STT service is ready or not + * This will be `false` when the app receives a SPEECH_TO_TEXT_DISCONNECTED exception + * and it cannot subscribe to STT + */ + isSttReady(): boolean { + return this._isSttReady.getValue(); + } + + /** + * @internal + */ + setSTTReady(value: boolean): void { + if (this._isSttReady.getValue() !== value) { + this._isSttReady.next(value); + } + } + /** * @internal */ @@ -210,7 +236,7 @@ export class OpenViduService { * @internal * Initialize a publisher checking devices saved on storage or if participant have devices available. */ - async initDefaultPublisher(): Promise { + async initDefaultPublisher(): Promise { const hasVideoDevices = this.deviceService.hasVideoDeviceAvailable(); const hasAudioDevices = this.deviceService.hasAudioDeviceAvailable(); const isVideoActive = !this.deviceService.isVideoMuted(); @@ -487,19 +513,67 @@ export class OpenViduService { } } - // private destroyPublisher(publisher: Publisher): void { - // if (!!publisher) { - // if (publisher.stream.getWebRtcPeer()) { - // publisher.stream.disposeWebRtcPeer(); - // } - // publisher.stream.disposeMediaStream(); - // if (publisher.id === this.participantService.getMyCameraPublisher().id) { - // this.participantService.setMyCameraPublisher(publisher); - // } else if (publisher.id === this.participantService.getMyScreenPublisher().id) { - // this.participantService.setMyScreenPublisher(publisher); - // } - // } - // } + /** + * @internal + * Subscribe all `CAMERA` stream types to speech-to-text + * It will retry the subscription each `STT_TIMEOUT_MS` + * + * @param lang The language of the Stream's audio track. + */ + async subscribeRemotesToSTT(lang: string): Promise { + const remoteParticipants = this.participantService.getRemoteParticipants(); + let successNumber = 0; + + for (const p of remoteParticipants) { + const stream = p.getCameraConnection()?.streamManager?.stream; + if (stream) { + try { + await this.subscribeStreamToStt(stream, lang); + successNumber++; + } catch (error) { + this.log.e(`Error subscribing ${stream.streamId} to STT:`, error); + break; + } + } + } + + this.setSTTReady(successNumber === remoteParticipants.length); + if (!this.isSttReady()) { + this.log.w('STT is not ready. Retrying subscription...'); + this.sttReconnectionTimeout = setTimeout(this.subscribeRemotesToSTT.bind(this, lang), this.STT_TIMEOUT_MS); + } + } + + /** + * @internal + * Subscribe a stream to speech-to-text + * @param stream + * @param lang + */ + async subscribeStreamToStt(stream: Stream, lang: string): Promise { + await this.getWebcamSession().subscribeToSpeechToText(stream, lang); + this.log.d(`Subscribed stream ${stream.streamId} to STT with ${lang} language.`); + } + + /** + * @internal + * Unsubscribe to all `CAMERA` stream types to speech-to-text if STT is up(ready) + */ + async unsubscribeRemotesFromSTT(): Promise { + clearTimeout(this.sttReconnectionTimeout); + if (this.isSttReady()) { + for (const p of this.participantService.getRemoteParticipants()) { + const stream = p.getCameraConnection().streamManager.stream; + if (stream) { + try { + await this.getWebcamSession().unsubscribeFromSpeechToText(stream); + } catch (error) { + this.log.e(`Error unsubscribing ${stream.streamId} from STT:`, error); + } + } + } + } + } private async createMediaStream(pp: PublisherProperties): Promise { let mediaStream: MediaStream;