From 488811f132c5852470f4813c18525df157dc3f9f Mon Sep 17 00:00:00 2001 From: Carlos Santos <4a.santos@gmail.com> Date: Fri, 22 Aug 2025 18:03:51 +0200 Subject: [PATCH] ov-components: Refactor device handling and improve error logging in audio and video components --- .../media-element/media-element.component.ts | 31 +++- .../pre-join/pre-join.component.html | 5 +- .../components/pre-join/pre-join.component.ts | 42 +++-- .../audio-devices/audio-devices.component.ts | 29 ++- .../video-devices/video-devices.component.ts | 50 +++-- .../videoconference.component.html | 2 +- .../src/lib/services/device/device.service.ts | 56 +++++- .../lib/services/openvidu/openvidu.service.ts | 172 +++++++++++++++++- 8 files changed, 307 insertions(+), 80 deletions(-) diff --git a/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/media-element/media-element.component.ts b/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/media-element/media-element.component.ts index 6df1e866..48095dd2 100644 --- a/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/media-element/media-element.component.ts +++ b/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/media-element/media-element.component.ts @@ -1,5 +1,5 @@ import { animate, style, transition, trigger } from '@angular/animations'; -import { AfterViewInit, Component, ElementRef, Input, ViewChild } from '@angular/core'; +import { AfterViewInit, Component, ElementRef, Input, OnDestroy, ViewChild } from '@angular/core'; import { Track } from 'livekit-client'; /** @@ -21,12 +21,13 @@ import { Track } from 'livekit-client'; ], standalone: false }) -export class MediaElementComponent implements AfterViewInit { +export class MediaElementComponent implements AfterViewInit, OnDestroy { _track: Track; _videoElement: ElementRef; _audioElement: ElementRef; type: Track.Source = Track.Source.Camera; private _muted: boolean = false; + private previousTrack: Track | null = null; @Input() showAvatar: boolean; @Input() avatarColor: string; @@ -37,20 +38,25 @@ export class MediaElementComponent implements AfterViewInit { set videoElement(element: ElementRef) { this._videoElement = element; this.attachTracks(); - } @ViewChild('audioElement', { static: false }) set audioElement(element: ElementRef) { this._audioElement = element; this.attachTracks(); - } @Input() set track(track: Track) { if (!track) return; + + // Detach previous track if it's different + if (this.previousTrack && this.previousTrack !== track) { + this.detachPreviousTrack(); + } + this._track = track; + this.previousTrack = track; this.attachTracks(); } @@ -69,6 +75,23 @@ export class MediaElementComponent implements AfterViewInit { }); } + ngOnDestroy() { + this.detachPreviousTrack(); + } + + private detachPreviousTrack() { + if (this.previousTrack) { + // Detach from video element + if (this.isVideoTrack() && this._videoElement?.nativeElement) { + this.previousTrack.detach(this._videoElement.nativeElement); + } + // Detach from audio element + if (this.isAudioTrack() && this._audioElement?.nativeElement) { + this.previousTrack.detach(this._audioElement.nativeElement); + } + } + } + private updateVideoStyles() { this.type = this._track.source; if (this.type === Track.Source.ScreenShare) { diff --git a/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/pre-join/pre-join.component.html b/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/pre-join/pre-join.component.html index d11b68d5..1ee1ac1a 100644 --- a/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/pre-join/pre-join.component.html +++ b/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/pre-join/pre-join.component.html @@ -23,11 +23,12 @@
@@ -37,7 +38,7 @@
compact', [animate('250ms cubic-bezier(0.25, 0.8, 0.25, 1)')]), @@ -58,15 +58,6 @@ import { LangOption } from '../../models/lang.model'; opacity: 1 }) ) - ]), - transition(':leave', [ - animate( - '200ms cubic-bezier(0.25, 0.46, 0.45, 0.94)', - style({ - opacity: 0, - transform: 'translateY(-10px)' - }) - ) ]) ]) ] @@ -127,7 +118,7 @@ export class PreJoinComponent implements OnInit, OnDestroy { async ngOnInit() { this.subscribeToPrejoinDirectives(); - await this.initializeDevices(); + await this.initializeDevicesWithRetry(); this.windowSize = window.innerWidth; this.isLoading = false; this.changeDetector.markForCheck(); @@ -150,10 +141,6 @@ export class PreJoinComponent implements OnInit, OnDestroy { } } - private async initializeDevices() { - await this.initializeDevicesWithRetry(); - } - onDeviceSelectorClicked() { // Some devices as iPhone do not show the menu panels correctly // Updating the container where the panel is added fix the problem. @@ -248,6 +235,27 @@ export class PreJoinComponent implements OnInit, OnDestroy { this.onVideoEnabledChanged.emit(enabled); } + async videoDeviceChanged(device: CustomDevice) { + try { + this.log.d('Video device changed to:', device); + + // Get the updated tracks from the service + const updatedTracks = this.openviduService.getLocalTracks(); + + // Find the new video track + const newVideoTrack = updatedTracks.find((track) => track.kind === 'video'); + + // if (newVideoTrack && newVideoTrack !== this.videoTrack) { + this.tracks = updatedTracks; + this.videoTrack = newVideoTrack; + + this.onVideoDeviceChanged.emit(device); + } catch (error) { + this.log.e('Error handling video device change:', error); + this.handleError(error); + } + } + onVideoDevicesLoaded(devices: CustomDevice[]) { this.hasVideoDevices = devices.length > 0; } diff --git a/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/settings/audio-devices/audio-devices.component.ts b/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/settings/audio-devices/audio-devices.component.ts index 203f1b27..4673c49d 100644 --- a/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/settings/audio-devices/audio-devices.component.ts +++ b/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/settings/audio-devices/audio-devices.component.ts @@ -5,6 +5,8 @@ import { DeviceService } from '../../../services/device/device.service'; import { ParticipantService } from '../../../services/participant/participant.service'; import { StorageService } from '../../../services/storage/storage.service'; import { ParticipantModel } from '../../../models/participant.model'; +import { LoggerService } from '../../../services/logger/logger.service'; +import { ILogger } from '../../../models/logger.model'; /** * @internal @@ -26,12 +28,16 @@ export class AudioDevicesComponent implements OnInit, OnDestroy { microphoneSelected: CustomDevice | undefined; microphones: CustomDevice[] = []; private localParticipantSubscription: Subscription; + private log: ILogger; constructor( private deviceSrv: DeviceService, private storageSrv: StorageService, - private participantService: ParticipantService - ) {} + private participantService: ParticipantService, + private loggerSrv: LoggerService + ) { + this.log = this.loggerSrv.get('AudioDevicesComponent'); + } async ngOnInit() { this.subscribeToParticipantMediaProperties(); @@ -60,14 +66,19 @@ export class AudioDevicesComponent implements OnInit, OnDestroy { } async onMicrophoneSelected(event: any) { - const device: CustomDevice = event?.value; - if (this.deviceSrv.needUpdateAudioTrack(device)) { - this.microphoneStatusChanging = true; - await this.participantService.switchMicrophone(device.device); - this.deviceSrv.setMicSelected(device.device); - this.microphoneSelected = this.deviceSrv.getMicrophoneSelected(); + try { + const device: CustomDevice = event?.value; + if (this.deviceSrv.needUpdateAudioTrack(device)) { + this.microphoneStatusChanging = true; + await this.participantService.switchMicrophone(device.device); + this.deviceSrv.setMicSelected(device.device); + this.microphoneSelected = this.deviceSrv.getMicrophoneSelected(); + this.onAudioDeviceChanged.emit(this.microphoneSelected); + } + } catch (error) { + this.log.e('Error switching microphone', error); + } finally { this.microphoneStatusChanging = false; - this.onAudioDeviceChanged.emit(this.microphoneSelected); } } diff --git a/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/settings/video-devices/video-devices.component.ts b/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/settings/video-devices/video-devices.component.ts index bf8559c9..8a56ba64 100644 --- a/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/settings/video-devices/video-devices.component.ts +++ b/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/settings/video-devices/video-devices.component.ts @@ -5,6 +5,8 @@ import { DeviceService } from '../../../services/device/device.service'; import { ParticipantService } from '../../../services/participant/participant.service'; import { StorageService } from '../../../services/storage/storage.service'; import { ParticipantModel } from '../../../models/participant.model'; +import { LoggerService } from '../../../services/logger/logger.service'; +import { ILogger } from '../../../models/logger.model'; /** * @internal @@ -28,11 +30,16 @@ export class VideoDevicesComponent implements OnInit, OnDestroy { cameras: CustomDevice[] = []; localParticipantSubscription: Subscription; + private log: ILogger; + constructor( private storageSrv: StorageService, private deviceSrv: DeviceService, - private participantService: ParticipantService - ) {} + private participantService: ParticipantService, + private loggerSrv: LoggerService + ) { + this.log = this.loggerSrv.get('VideoDevicesComponent'); + } async ngOnInit() { this.subscribeToParticipantMediaProperties(); @@ -63,37 +70,24 @@ export class VideoDevicesComponent implements OnInit, OnDestroy { } async onCameraSelected(event: any) { - const device: CustomDevice = event?.value; + try { + const device: CustomDevice = event?.value; - // Is New deviceId different from the old one? - if (this.deviceSrv.needUpdateVideoTrack(device)) { - // const mirror = this.deviceSrv.cameraNeedsMirror(device.device); - // Reapply Virtual Background to new Publisher if necessary - // const backgroundSelected = this.backgroundService.backgroundSelected.getValue(); - // const isBackgroundApplied = this.backgroundService.isBackgroundApplied(); + // Is New deviceId different from the old one? + if (this.deviceSrv.needUpdateVideoTrack(device)) { - // if (isBackgroundApplied) { - // await this.backgroundService.removeBackground(); - // } - // const pp: PublisherProperties = { videoSource: device.device, audioSource: false, mirror }; - // const publisher = this.participantService.getMyCameraPublisher(); - // await this.openviduService.replaceCameraTrack(publisher, pp); + this.cameraStatusChanging = true; - this.cameraStatusChanging = true; + await this.participantService.switchCamera(device.device); - await this.participantService.switchCamera(device.device); - - // if (isBackgroundApplied) { - // const bgSelected = this.backgroundService.backgrounds.find((b) => b.id === backgroundSelected); - // if (bgSelected) { - // await this.backgroundService.applyBackground(bgSelected); - // } - // } - - this.deviceSrv.setCameraSelected(device.device); - this.cameraSelected = this.deviceSrv.getCameraSelected(); + this.deviceSrv.setCameraSelected(device.device); + this.cameraSelected = device; + this.onVideoDeviceChanged.emit(this.cameraSelected); + } + } catch (error) { + this.log.e('Error switching camera', error); + } finally { this.cameraStatusChanging = false; - this.onVideoDeviceChanged.emit(this.cameraSelected); } } diff --git a/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/videoconference/videoconference.component.html b/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/videoconference/videoconference.component.html index 9f32527b..89f616e5 100644 --- a/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/videoconference/videoconference.component.html +++ b/openvidu-components-angular/projects/openvidu-components-angular/src/lib/components/videoconference/videoconference.component.html @@ -1,7 +1,7 @@
@if (componentState.isLoading) { -
+
{{ 'PREJOIN.PREPARING' | translate }}
diff --git a/openvidu-components-angular/projects/openvidu-components-angular/src/lib/services/device/device.service.ts b/openvidu-components-angular/projects/openvidu-components-angular/src/lib/services/device/device.service.ts index 7756be0c..574fb0c6 100644 --- a/openvidu-components-angular/projects/openvidu-components-angular/src/lib/services/device/device.service.ts +++ b/openvidu-components-angular/projects/openvidu-components-angular/src/lib/services/device/device.service.ts @@ -180,18 +180,33 @@ export class DeviceService { */ private async getLocalDevices(): Promise { // Forcing media permissions request. - let localTracks: LocalTrack[] = []; - try { - localTracks = await createLocalTracks({ audio: true, video: true }); - localTracks.forEach((track) => track.stop()); + const strategies = [ + { audio: true, video: true }, + { audio: true, video: false }, + { audio: false, video: true } + ]; - const devices = this.platformSrv.isFirefox() ? await this.getMediaDevicesFirefox() : await Room.getLocalDevices(); - return devices.filter((d: MediaDeviceInfo) => d.label && d.deviceId && d.deviceId !== 'default'); - } catch (error) { - this.log.e('Error getting local devices', error); - this.deviceAccessDeniedError = true; - return []; + for (const strategy of strategies) { + try { + this.log.d(`Trying strategy: audio=${strategy.audio}, video=${strategy.video}`); + const localTracks = await createLocalTracks(strategy); + localTracks.forEach((track) => track.stop()); + + // Permission granted + const devices = this.platformSrv.isFirefox() ? await this.getMediaDevicesFirefox() : await Room.getLocalDevices(); + + return devices.filter((d: MediaDeviceInfo) => d.label && d.deviceId && d.deviceId !== 'default'); + } catch (error: any) { + this.log.w(`Strategy failed: audio=${strategy.audio}, video=${strategy.video}`, error); + + // If it's the last attempt and failed, we handle the error + if (strategy === strategies[strategies.length - 1]) { + return await this.handleFinalFallback(error); + } + } } + + return []; } private async getMediaDevicesFirefox(): Promise { @@ -199,4 +214,25 @@ export class DeviceService { await navigator.mediaDevices.getUserMedia({ audio: true, video: true }); return navigator.mediaDevices.enumerateDevices(); } + + private async handleFinalFallback(error: any): Promise { + this.log.w('All permission strategies failed, trying device enumeration without permissions'); + + try { + if (error?.name === 'NotReadableError' || error?.name === 'AbortError') { + this.log.w('Device busy, using enumerateDevices() instead'); + const devices = await navigator.mediaDevices.enumerateDevices(); + return devices.filter((d) => d.deviceId && d.deviceId !== 'default'); + } + if (error?.name === 'NotAllowedError' || error?.name === 'SecurityError') { + this.log.w('Permission denied to access devices'); + this.deviceAccessDeniedError = true; + } + return []; + } catch (error) { + this.log.e('Complete failure getting devices', error); + this.deviceAccessDeniedError = true; + return []; + } + } } diff --git a/openvidu-components-angular/projects/openvidu-components-angular/src/lib/services/openvidu/openvidu.service.ts b/openvidu-components-angular/projects/openvidu-components-angular/src/lib/services/openvidu/openvidu.service.ts index 4f6ee3e7..1174c84b 100644 --- a/openvidu-components-angular/projects/openvidu-components-angular/src/lib/services/openvidu/openvidu.service.ts +++ b/openvidu-components-angular/projects/openvidu-components-angular/src/lib/services/openvidu/openvidu.service.ts @@ -208,7 +208,7 @@ export class OpenViduService { * @internal */ setLocalTracks(tracks: LocalTrack[]): void { - this.localTracks = tracks; + this.localTracks = tracks.filter((track) => track !== undefined) as LocalTrack[]; } /** @@ -235,12 +235,14 @@ export class OpenViduService { * * @param videoDeviceId - The ID of the video device to use. If not provided, the default video device will be used. * @param audioDeviceId - The ID of the audio device to use. If not provided, the default audio device will be used. + * @param allowPartialCreation - If true, allows creating tracks even if some devices fail * @returns A promise that resolves to an array of LocalTrack objects representing the created tracks. * @internal */ async createLocalTracks( videoDeviceId: string | boolean | undefined = undefined, - audioDeviceId: string | boolean | undefined = undefined + audioDeviceId: string | boolean | undefined = undefined, + allowPartialCreation: boolean = true ): Promise { // Default values: true if device is enabled, false otherwise videoDeviceId ??= this.deviceService.isCameraEnabled(); @@ -277,9 +279,17 @@ export class OpenViduService { } let newLocalTracks: LocalTrack[] = []; + if (options.audio || options.video) { this.log.d('Creating local tracks with options', options); - newLocalTracks = await createLocalTracks(options); + + if (allowPartialCreation) { + // Try to create tracks separately to handle device conflicts gracefully + newLocalTracks = await this.createTracksWithFallback(options); + } else { + // Original behavior - all or nothing + newLocalTracks = await createLocalTracks(options); + } // Mute tracks if devices are disabled if (!this.deviceService.isCameraEnabled()) { @@ -292,6 +302,41 @@ export class OpenViduService { return newLocalTracks; } + /** + * Creates tracks with fallback strategy to handle device conflicts + * @param options - The track creation options + * @returns Array of successfully created tracks + * @internal + */ + private async createTracksWithFallback(options: CreateLocalTracksOptions): Promise { + const tracks: LocalTrack[] = []; + + // Try to create video track separately + if (options.video) { + try { + const videoTracks = await createLocalTracks({ video: options.video }); + tracks.push(...videoTracks); + this.log.d('Video track created successfully'); + } catch (error) { + this.log.w('Failed to create video track, device may be busy:', error); + // Still continue to try audio track + } + } + + // Try to create audio track separately + if (options.audio) { + try { + const audioTracks = await createLocalTracks({ audio: options.audio }); + tracks.push(...audioTracks); + this.log.d('Audio track created successfully'); + } catch (error) { + this.log.w('Failed to create audio track, device may be busy:', error); + } + } + + return tracks; + } + /** * @internal * As the Room is not created yet, we need to handle the media tracks with a temporary array of tracks. @@ -349,21 +394,130 @@ export class OpenViduService { } /** - * Switch the microphone device when the room is not connected (prejoin page) + * Switch the camera device when the room is not connected (prejoin page) * @param deviceId new video device to use * @internal */ - switchCamera(deviceId: string): Promise { - return (this.localTracks?.find((track) => track.kind === Track.Kind.Video) as LocalVideoTrack).restartTrack({ deviceId: deviceId }); + async switchCamera(deviceId: string): Promise { + const existingTrack = this.localTracks.find((track) => track.kind === Track.Kind.Video) as LocalVideoTrack; + + if (existingTrack) { + //TODO: SHould use replace track using restartTrack + // Try to restart existing track + this.removeVideoTrack(); + // try { + // await existingTrack.restartTrack({ deviceId: deviceId }); + // this.log.d('Camera switched successfully using existing track'); + // return; + // } catch (error) { + // this.log.w('Failed to restart video track, trying to create new one:', error); + // // Remove the failed track + // this.removeVideoTrack(); + // } + } + + // Create new video track if no existing track or restart failed + try { + const newVideoTracks = await createLocalTracks({ + video: { deviceId: deviceId } + }); + + const videoTrack = newVideoTracks.find((t) => t.kind === Track.Kind.Video); + if (videoTrack) { + + // Mute if camera is disabled in settings + if (!this.deviceService.isCameraEnabled()) { + await videoTrack.mute(); + } + + this.localTracks.push(videoTrack); + this.log.d('New camera track created and added'); + } + } catch (error) { + this.log.e('Failed to create new video track:', error); + throw new Error(`Failed to switch camera: ${error.message}`); + } } /** * Switches the microphone device when the room is not connected (prejoin page) - * @param deviceId new video device to use + * @param deviceId new audio device to use * @internal */ - switchMicrophone(deviceId: string): Promise { - return (this.localTracks?.find((track) => track.kind === Track.Kind.Audio) as LocalAudioTrack).restartTrack({ deviceId: deviceId }); + async switchMicrophone(deviceId: string): Promise { + const existingTrack = this.localTracks?.find((track) => track.kind === Track.Kind.Audio) as LocalAudioTrack; + + if (existingTrack) { + this.removeAudioTrack(); + //TODO: SHould use replace track using restartTrack + // Try to restart existing track + // try { + // await existingTrack.restartTrack({ deviceId: deviceId }); + // this.log.d('Microphone switched successfully using existing track'); + // return; + // } catch (error) { + // this.log.w('Failed to restart audio track, trying to create new one:', error); + // // Remove the failed track + // this.removeAudioTrack(); + // } + } + + // Create new audio track if no existing track or restart failed + try { + const newAudioTracks = await createLocalTracks({ + audio: { + deviceId: deviceId, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true + } + }); + + const audioTrack = newAudioTracks.find((t) => t.kind === Track.Kind.Audio); + if (audioTrack) { + this.localTracks.push(audioTrack); + + // Mute if microphone is disabled in settings + if (!this.deviceService.isMicrophoneEnabled()) { + await audioTrack.mute(); + } + + this.log.d('New microphone track created and added'); + } + } catch (error) { + this.log.e('Failed to create new audio track:', error); + throw new Error(`Failed to switch microphone: ${error.message}`); + } + } + + /** + * Removes video track from local tracks + * @internal + */ + private removeVideoTrack(): void { + const videoTrackIndex = this.localTracks.findIndex((track) => track.kind === Track.Kind.Video); + if (videoTrackIndex !== -1) { + const videoTrack = this.localTracks[videoTrackIndex]; + videoTrack.stop(); + videoTrack.detach(); + this.localTracks.splice(videoTrackIndex, 1); + this.log.d('Video track removed'); + } + } + + /** + * Removes audio track from local tracks + * @internal + */ + private removeAudioTrack(): void { + const audioTrackIndex = this.localTracks.findIndex((track) => track.kind === Track.Kind.Audio); + if (audioTrackIndex !== -1) { + const audioTrack = this.localTracks[audioTrackIndex]; + audioTrack.stop(); + audioTrack.detach(); + this.localTracks.splice(audioTrackIndex, 1); + this.log.d('Audio track removed'); + } } /**