From a8e0fcfb790e1a7f0e452e1ac166accc98cfd72f Mon Sep 17 00:00:00 2001 From: Carlos Santos <4a.santos@gmail.com> Date: Tue, 8 Nov 2022 15:29:06 +0100 Subject: [PATCH] openvidu-components: Subscribe to STT when captions opened --- .../components/captions/captions.component.ts | 34 ++++++++++++++++--- .../components/session/session.component.ts | 12 ++++--- .../lib/services/caption/caption.service.ts | 10 ++++++ 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.ts b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.ts index e5888420..0cbae2db 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.ts +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.ts @@ -51,6 +51,7 @@ export class CaptionsComponent implements OnInit { session: Session; private deleteTimeout: NodeJS.Timeout; + private deleteAllTimeout: NodeJS.Timeout; private DELETE_TIMEOUT = 5 * 1000; private MAX_EVENTS_LIMIT = 3; @@ -67,16 +68,29 @@ export class CaptionsComponent implements OnInit { private cd: ChangeDetectorRef ) {} - ngOnInit(): void { + async ngOnInit(): Promise { + this.captionService.setCaptionsEnabled(true); this.captionLangSelected = this.captionService.getLangSelected(); this.session = this.openviduService.getWebcamSession(); + for (const p of this.participantService.getRemoteParticipants()) { + const stream = p.getCameraConnection().streamManager.stream; + await this.session.subscribeToSpeechToText(stream, this.captionLangSelected.ISO); + } + this.subscribeToCaptionLanguage(); this.subscribeToPanelToggling(); this.subscribeToTranscription(); } - ngOnDestroy() { + async ngOnDestroy() { + this.captionService.setCaptionsEnabled(false); + + for (const p of this.participantService.getRemoteParticipants()) { + const stream = p.getCameraConnection().streamManager.stream; + await this.session.unsubscribeFromSpeechToText(stream); + } + if (this.screenSizeSub) this.screenSizeSub.unsubscribe(); if (this.panelTogglingSubscription) this.panelTogglingSubscription.unsubscribe(); this.session.off('speechToTextMessage'); @@ -89,6 +103,7 @@ export class CaptionsComponent implements OnInit { private subscribeToTranscription() { this.session.on('speechToTextMessage', (event: SpeechToTextEvent) => { + clearInterval(this.deleteAllTimeout); const { connectionId, data } = event.connection; const nickname: string = this.participantService.getNicknameFromConnectionData(data); const color = this.participantService.getRemoteParticipantByConnectionId(connectionId)?.colorProfile || ''; @@ -101,6 +116,8 @@ export class CaptionsComponent implements OnInit { type: event.reason }; this.updateCaption(caption); + // Delete all events when there are no more events for a period of time + this.deleteAllEventsAfterDelay(this.DELETE_TIMEOUT); this.cd.markForCheck(); }); } @@ -127,7 +144,7 @@ export class CaptionsComponent implements OnInit { clearInterval(this.deleteTimeout); } captionEventsCopy.push(caption); - this.deleteEventAfterDelay(this.DELETE_TIMEOUT); + this.deleteFirstEventAfterDelay(this.DELETE_TIMEOUT); } else { lastCaption.text = caption.text; lastCaption.type = caption.type; @@ -139,7 +156,7 @@ export class CaptionsComponent implements OnInit { clearInterval(this.deleteTimeout); } captionEventsCopy.push(caption); - this.deleteEventAfterDelay(this.DELETE_TIMEOUT); + this.deleteFirstEventAfterDelay(this.DELETE_TIMEOUT); } } @@ -147,13 +164,20 @@ export class CaptionsComponent implements OnInit { this.scrollToBottom(); } - private deleteEventAfterDelay(timeout: number) { + private deleteFirstEventAfterDelay(timeout: number) { this.deleteTimeout = setTimeout(() => { this.captionEvents.shift(); this.cd.markForCheck(); }, timeout); } + private deleteAllEventsAfterDelay(timeout: number) { + this.deleteAllTimeout = setTimeout(() => { + this.captionEvents = []; + this.cd.markForCheck(); + }, timeout); + } + private subscribeToCaptionLanguage() { this.captionLanguageSubscription = this.captionService.captionLangObs.subscribe((lang) => { this.captionLangSelected = lang; diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/session/session.component.ts b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/session/session.component.ts index e6531c5f..53f35513 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/components/session/session.component.ts +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/components/session/session.component.ts @@ -315,7 +315,7 @@ export class SessionComponent implements OnInit, OnDestroy { this.participantService.addRemoteConnection(connectionId, data, subscriber); // this.oVSessionService.sendNicknameSignal(event.stream.connection); - if (this.participantService.getTypeConnectionData(data) === VideoType.CAMERA) { + if (this.captionService.areCaptionsEnabled() && this.participantService.getTypeConnectionData(data) === VideoType.CAMERA) { // Only subscribe to STT when stream is CAMERA type and it is a remote stream try { await this.session.subscribeToSpeechToText(event.stream, this.captionService.getLangSelected().ISO); @@ -331,10 +331,12 @@ export class SessionComponent implements OnInit, OnDestroy { this.session.on('streamDestroyed', async (event: StreamEvent) => { const connectionId = event.stream.connection.connectionId; this.participantService.removeConnectionByConnectionId(connectionId); - try { - await this.session.unsubscribeFromSpeechToText(event.stream); - } catch (error) { - this.log.e('Error unsubscribing from STT: ', error); + if(this.captionService.areCaptionsEnabled()){ + try { + await this.session.unsubscribeFromSpeechToText(event.stream); + } catch (error) { + this.log.e('Error unsubscribing from STT: ', error); + } } }); } diff --git a/openvidu-components-angular/projects/openvidu-angular/src/lib/services/caption/caption.service.ts b/openvidu-components-angular/projects/openvidu-angular/src/lib/services/caption/caption.service.ts index 9fc4768d..bb501ed4 100644 --- a/openvidu-components-angular/projects/openvidu-angular/src/lib/services/caption/caption.service.ts +++ b/openvidu-components-angular/projects/openvidu-angular/src/lib/services/caption/caption.service.ts @@ -9,6 +9,7 @@ import { StorageService } from '../storage/storage.service'; providedIn: 'root' }) export class CaptionService { + private langTitles = [ { name: 'English', ISO: 'en-US' }, { name: 'Español', ISO: 'es-ES' }, @@ -23,6 +24,7 @@ export class CaptionService { captionLangSelected: { name: string; ISO: string }; captionLangObs: Observable<{ name: string; ISO: string }>; private _captionLangObs: Subject<{ name: string; ISO: string }> = new Subject(); + private captionsEnabled: boolean = false; constructor(private storageService: StorageService) { const iso = this.storageService.getCaptionsLang(); @@ -32,6 +34,14 @@ export class CaptionService { this.captionLangObs = this._captionLangObs.asObservable(); } + setCaptionsEnabled(value: boolean) { + this.captionsEnabled = value; + } + + areCaptionsEnabled(): boolean { + return this.captionsEnabled; + } + setLanguage(lang: string) { if (this.langTitles.some((l) => l.ISO === lang)) { this.captionLangSelected = this.langTitles.find((l) => l.ISO === lang);