ambiser
/
openvidu
mirror of https://github.com/OpenVidu/openvidu.git
8
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

openvidu-components: Subscribe to STT when captions opened

pull/758/head
Carlos Santos 2022-11-08 15:29:06 +01:00
parent c769ddd02d
commit a8e0fcfb79
3 changed files with 46 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
openvidu-components-angular/projects/openvidu-angular/src/lib/components/captions/captions.component.ts
View File

@ -51,6 +51,7 @@ export class CaptionsComponent implements OnInit {
session: Session;
private deleteTimeout: NodeJS.Timeout;
private deleteAllTimeout: NodeJS.Timeout;
private DELETE_TIMEOUT = 5 * 1000;
private MAX_EVENTS_LIMIT = 3;
@ -67,16 +68,29 @@ export class CaptionsComponent implements OnInit {
private cd: ChangeDetectorRef
) {}
ngOnInit(): void {
async ngOnInit(): Promise<void> {
this.captionService.setCaptionsEnabled(true);
this.captionLangSelected = this.captionService.getLangSelected();
this.session = this.openviduService.getWebcamSession();
for (const p of this.participantService.getRemoteParticipants()) {
const stream = p.getCameraConnection().streamManager.stream;
await this.session.subscribeToSpeechToText(stream, this.captionLangSelected.ISO);
}
this.subscribeToCaptionLanguage();
this.subscribeToPanelToggling();
this.subscribeToTranscription();
}
ngOnDestroy() {
async ngOnDestroy() {
this.captionService.setCaptionsEnabled(false);
for (const p of this.participantService.getRemoteParticipants()) {
const stream = p.getCameraConnection().streamManager.stream;
await this.session.unsubscribeFromSpeechToText(stream);
}
if (this.screenSizeSub) this.screenSizeSub.unsubscribe();
if (this.panelTogglingSubscription) this.panelTogglingSubscription.unsubscribe();
this.session.off('speechToTextMessage');
@ -89,6 +103,7 @@ export class CaptionsComponent implements OnInit {
private subscribeToTranscription() {
this.session.on('speechToTextMessage', (event: SpeechToTextEvent) => {
clearInterval(this.deleteAllTimeout);
const { connectionId, data } = event.connection;
const nickname: string = this.participantService.getNicknameFromConnectionData(data);
const color = this.participantService.getRemoteParticipantByConnectionId(connectionId)?.colorProfile || '';
@ -101,6 +116,8 @@ export class CaptionsComponent implements OnInit {
type: event.reason
};
this.updateCaption(caption);
// Delete all events when there are no more events for a period of time
this.deleteAllEventsAfterDelay(this.DELETE_TIMEOUT);
this.cd.markForCheck();
});
}
@ -127,7 +144,7 @@ export class CaptionsComponent implements OnInit {
clearInterval(this.deleteTimeout);
}
captionEventsCopy.push(caption);
this.deleteEventAfterDelay(this.DELETE_TIMEOUT);
this.deleteFirstEventAfterDelay(this.DELETE_TIMEOUT);
} else {
lastCaption.text = caption.text;
lastCaption.type = caption.type;
@ -139,7 +156,7 @@ export class CaptionsComponent implements OnInit {
clearInterval(this.deleteTimeout);
}
captionEventsCopy.push(caption);
this.deleteEventAfterDelay(this.DELETE_TIMEOUT);
this.deleteFirstEventAfterDelay(this.DELETE_TIMEOUT);
}
}
@ -147,13 +164,20 @@ export class CaptionsComponent implements OnInit {
this.scrollToBottom();
}
private deleteEventAfterDelay(timeout: number) {
private deleteFirstEventAfterDelay(timeout: number) {
this.deleteTimeout = setTimeout(() => {
this.captionEvents.shift();
this.cd.markForCheck();
}, timeout);
}
private deleteAllEventsAfterDelay(timeout: number) {
this.deleteAllTimeout = setTimeout(() => {
this.captionEvents = [];
this.cd.markForCheck();
}, timeout);
}
private subscribeToCaptionLanguage() {
this.captionLanguageSubscription = this.captionService.captionLangObs.subscribe((lang) => {
this.captionLangSelected = lang;

4
openvidu-components-angular/projects/openvidu-angular/src/lib/components/session/session.component.ts
View File

@ -315,7 +315,7 @@ export class SessionComponent implements OnInit, OnDestroy {
this.participantService.addRemoteConnection(connectionId, data, subscriber);
// this.oVSessionService.sendNicknameSignal(event.stream.connection);
if (this.participantService.getTypeConnectionData(data) === VideoType.CAMERA) {
if (this.captionService.areCaptionsEnabled() && this.participantService.getTypeConnectionData(data) === VideoType.CAMERA) {
// Only subscribe to STT when stream is CAMERA type and it is a remote stream
try {
await this.session.subscribeToSpeechToText(event.stream, this.captionService.getLangSelected().ISO);
@ -331,11 +331,13 @@ export class SessionComponent implements OnInit, OnDestroy {
this.session.on('streamDestroyed', async (event: StreamEvent) => {
const connectionId = event.stream.connection.connectionId;
this.participantService.removeConnectionByConnectionId(connectionId);
if(this.captionService.areCaptionsEnabled()){
try {
await this.session.unsubscribeFromSpeechToText(event.stream);
} catch (error) {
this.log.e('Error unsubscribing from STT: ', error);
}
}
});
}

10
openvidu-components-angular/projects/openvidu-angular/src/lib/services/caption/caption.service.ts
View File

@ -9,6 +9,7 @@ import { StorageService } from '../storage/storage.service';
providedIn: 'root'
})
export class CaptionService {
private langTitles = [
{ name: 'English', ISO: 'en-US' },
{ name: 'Español', ISO: 'es-ES' },
@ -23,6 +24,7 @@ export class CaptionService {
captionLangSelected: { name: string; ISO: string };
captionLangObs: Observable<{ name: string; ISO: string }>;
private _captionLangObs: Subject<{ name: string; ISO: string }> = new Subject();
private captionsEnabled: boolean = false;
constructor(private storageService: StorageService) {
const iso = this.storageService.getCaptionsLang();
@ -32,6 +34,14 @@ export class CaptionService {
this.captionLangObs = this._captionLangObs.asObservable();
}
setCaptionsEnabled(value: boolean) {
this.captionsEnabled = value;
}
areCaptionsEnabled(): boolean {
return this.captionsEnabled;
}
setLanguage(lang: string) {
if (this.langTitles.some((l) => l.ISO === lang)) {
this.captionLangSelected = this.langTitles.find((l) => l.ISO === lang);