From edf68d16b748d230c5651d06b39f4836bdfd17cc Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 30 Sep 2025 11:33:45 +0200 Subject: [PATCH 01/46] refactoring: prep work extract to file + documentation --- src/state/CallViewModel.ts | 3 +- src/state/Connection.ts | 281 ++++++++------------------------- src/state/PublishConnection.ts | 224 ++++++++++++++++++++++++++ 3 files changed, 295 insertions(+), 213 deletions(-) create mode 100644 src/state/PublishConnection.ts diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 7e3a5bdf8..b6327cfa0 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -122,11 +122,12 @@ import { } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { Connection, PublishConnection } from "./Connection"; +import { Connection } from "./Connection"; import { type MuteStates } from "./MuteStates"; import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../widget"; +import { PublishConnection } from "./PublishConnection.ts"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; diff --git a/src/state/Connection.ts b/src/state/Connection.ts index db456ba03..f725ddda2 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -5,55 +5,50 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - connectedParticipantsObserver, - connectionStateObserver, -} from "@livekit/components-core"; -import { - ConnectionState, - Room as LivekitRoom, - type E2EEOptions, - Track, - LocalVideoTrack, -} from "livekit-client"; +import { connectedParticipantsObserver, connectionStateObserver } from "@livekit/components-core"; +import { type ConnectionState, type E2EEOptions, Room as LivekitRoom } from "livekit-client"; import { type MatrixClient } from "matrix-js-sdk"; -import { - type LivekitFocus, - type CallMembership, -} from "matrix-js-sdk/lib/matrixrtc"; -import { - combineLatest, - map, - NEVER, - type Observable, - type Subscription, - switchMap, -} from "rxjs"; -import { logger } from "matrix-js-sdk/lib/logger"; +import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; +import { combineLatest } from "rxjs"; -import { type SelectedDevice, type MediaDevices } from "./MediaDevices"; import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; import { type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; import { defaultLiveKitOptions } from "../livekit/options"; -import { getValue } from "../utils/observable"; -import { getUrlParams } from "../UrlParams"; -import { type MuteStates } from "./MuteStates"; -import { - type ProcessorState, - trackProcessorSync, -} from "../livekit/TrackProcessorContext"; -import { observeTrackReference$ } from "./MediaViewModel"; +/** + * A connection to a Matrix RTC LiveKit backend. + * + * Expose observables for participants and connection state. + */ export class Connection { + + /** + * Whether the connection has been stopped. + * @see Connection.stop + * */ protected stopped = false; + /** + * Starts the connection. + * + * This will: + * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.) + * 2. Use this token to request the SFU config to the MatrixRtc authentication service. + * 3. Connect to the configured LiveKit room. + */ public async start(): Promise { this.stopped = false; const { url, jwt } = await this.sfuConfig; if (!this.stopped) await this.livekitRoom.connect(url, jwt); } + /** + * Stops the connection. + * + * This will disconnect from the LiveKit room. + * If the connection is already stopped, this is a no-op. + */ public stop(): void { if (this.stopped) return; void this.livekitRoom.disconnect(); @@ -63,16 +58,47 @@ export class Connection { protected readonly sfuConfig = getSFUConfigWithOpenID( this.client, this.focus.livekit_service_url, - this.livekitAlias, + this.focus.livekit_alias ); - public readonly participantsIncludingSubscribers$; + /* + * An observable of the participants in the livekit room, including subscribers. + * Converts the livekit room events ParticipantConnected/ParticipantDisconnected/StateChange to an observable. + */ + protected readonly participantsIncludingSubscribers$; + + /** + * An observable of the participants that are publishing on this connection. + * This is derived from `participantsIncludingSubscribers$` and `membershipsFocusMap$`. + * It filters the participants to only those that are associated with a membership that claims to publish on this connection. + */ public readonly publishingParticipants$; + + /** + * The LiveKit room instance. + */ public readonly livekitRoom: LivekitRoom; + /** + * An observable of the livekit connection state. + * Converts the livekit room events StateChange to an observable. + */ public connectionState$: Behavior; + + /** + * Creates a new connection to a matrix RTC LiveKit backend. + * + * @param livekitRoom - Optional LiveKit room instance to use. If not provided, a new instance will be created. + * @param focus - The focus server to connect to. + * @param livekitAlias - The livekit alias to use when connecting to the focus server. TODO duplicate of focus? + * @param client - The matrix client, used to fetch the OpenId token. TODO refactor to avoid passing the whole client + * @param scope - The observable scope to use for creating observables. + * @param membershipsFocusMap$ - The observable of the current call RTC memberships and their associated focus. + * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. TODO refactor to avoid passing the whole options? + */ public constructor( protected readonly focus: LivekitFocus, + // TODO : remove livekitAlias, it's already in focus? protected readonly livekitAlias: string, protected readonly client: MatrixClient, protected readonly scope: ObservableScope, @@ -80,17 +106,17 @@ export class Connection { { membership: CallMembership; focus: LivekitFocus }[] >, e2eeLivekitOptions: E2EEOptions | undefined, - livekitRoom: LivekitRoom | undefined = undefined, + livekitRoom: LivekitRoom | undefined = undefined ) { this.livekitRoom = livekitRoom ?? new LivekitRoom({ ...defaultLiveKitOptions, - e2ee: e2eeLivekitOptions, + e2ee: e2eeLivekitOptions }); this.participantsIncludingSubscribers$ = this.scope.behavior( connectedParticipantsObserver(this.livekitRoom), - [], + [] ); this.publishingParticipants$ = this.scope.behavior( @@ -102,193 +128,24 @@ export class Connection { .flatMap(({ membership, focus }) => focus.livekit_service_url === this.focus.livekit_service_url ? [membership] - : [], + : [] ) // Find all associated publishing livekit participant objects .flatMap((membership) => { const participant = participants.find( (p) => - p.identity === `${membership.sender}:${membership.deviceId}`, + p.identity === `${membership.sender}:${membership.deviceId}` ); return participant ? [{ participant, membership }] : []; - }), + }) ), - [], + [] ); this.connectionState$ = this.scope.behavior( - connectionStateObserver(this.livekitRoom), + connectionStateObserver(this.livekitRoom) ); this.scope.onEnd(() => this.stop()); } } -export class PublishConnection extends Connection { - public async start(): Promise { - this.stopped = false; - const { url, jwt } = await this.sfuConfig; - if (!this.stopped) await this.livekitRoom.connect(url, jwt); - - if (!this.stopped) { - const tracks = await this.livekitRoom.localParticipant.createTracks({ - audio: this.muteStates.audio.enabled$.value, - video: this.muteStates.video.enabled$.value, - }); - for (const track of tracks) { - await this.livekitRoom.localParticipant.publishTrack(track); - } - } - } - - public constructor( - focus: LivekitFocus, - livekitAlias: string, - client: MatrixClient, - scope: ObservableScope, - membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitFocus }[] - >, - devices: MediaDevices, - private readonly muteStates: MuteStates, - e2eeLivekitOptions: E2EEOptions | undefined, - trackerProcessorState$: Behavior, - ) { - logger.info("[LivekitRoom] Create LiveKit room"); - const { controlledAudioDevices } = getUrlParams(); - - const room = new LivekitRoom({ - ...defaultLiveKitOptions, - videoCaptureDefaults: { - ...defaultLiveKitOptions.videoCaptureDefaults, - deviceId: devices.videoInput.selected$.value?.id, - processor: trackerProcessorState$.value.processor, - }, - audioCaptureDefaults: { - ...defaultLiveKitOptions.audioCaptureDefaults, - deviceId: devices.audioInput.selected$.value?.id, - }, - audioOutput: { - // When using controlled audio devices, we don't want to set the - // deviceId here, because it will be set by the native app. - // (also the id does not need to match a browser device id) - deviceId: controlledAudioDevices - ? undefined - : getValue(devices.audioOutput.selected$)?.id, - }, - e2ee: e2eeLivekitOptions, - }); - room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => { - logger.error("Failed to set E2EE enabled on room", e); - }); - - super( - focus, - livekitAlias, - client, - scope, - membershipsFocusMap$, - e2eeLivekitOptions, - room, - ); - - // Setup track processor syncing (blur) - const track$ = this.scope.behavior( - observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( - map((trackRef) => { - const track = trackRef?.publication?.track; - return track instanceof LocalVideoTrack ? track : null; - }), - ), - ); - trackProcessorSync(track$, trackerProcessorState$); - - this.muteStates.audio.setHandler(async (desired) => { - try { - await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); - } catch (e) { - logger.error("Failed to update LiveKit audio input mute state", e); - } - return this.livekitRoom.localParticipant.isMicrophoneEnabled; - }); - this.muteStates.video.setHandler(async (desired) => { - try { - await this.livekitRoom.localParticipant.setCameraEnabled(desired); - } catch (e) { - logger.error("Failed to update LiveKit video input mute state", e); - } - return this.livekitRoom.localParticipant.isCameraEnabled; - }); - this.scope.onEnd(() => { - this.muteStates.audio.unsetHandler(); - this.muteStates.video.unsetHandler(); - }); - - const syncDevice = ( - kind: MediaDeviceKind, - selected$: Observable, - ): Subscription => - selected$.pipe(this.scope.bind()).subscribe((device) => { - if (this.connectionState$.value !== ConnectionState.Connected) return; - logger.info( - "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", - this.livekitRoom.getActiveDevice(kind), - " !== ", - device?.id, - ); - if ( - device !== undefined && - this.livekitRoom.getActiveDevice(kind) !== device.id - ) { - this.livekitRoom - .switchActiveDevice(kind, device.id) - .catch((e) => - logger.error(`Failed to sync ${kind} device with LiveKit`, e), - ); - } - }); - - syncDevice("audioinput", devices.audioInput.selected$); - if (!controlledAudioDevices) - syncDevice("audiooutput", devices.audioOutput.selected$); - syncDevice("videoinput", devices.videoInput.selected$); - // Restart the audio input track whenever we detect that the active media - // device has changed to refer to a different hardware device. We do this - // for the sake of Chrome, which provides a "default" device that is meant - // to match the system's default audio input, whatever that may be. - // This is special-cased for only audio inputs because we need to dig around - // in the LocalParticipant object for the track object and there's not a nice - // way to do that generically. There is usually no OS-level default video capture - // device anyway, and audio outputs work differently. - devices.audioInput.selected$ - .pipe( - switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), - this.scope.bind(), - ) - .subscribe(() => { - if (this.connectionState$.value !== ConnectionState.Connected) return; - const activeMicTrack = Array.from( - this.livekitRoom.localParticipant.audioTrackPublications.values(), - ).find((d) => d.source === Track.Source.Microphone)?.track; - - if ( - activeMicTrack && - // only restart if the stream is still running: LiveKit will detect - // when a track stops & restart appropriately, so this is not our job. - // Plus, we need to avoid restarting again if the track is already in - // the process of being restarted. - activeMicTrack.mediaStreamTrack.readyState !== "ended" - ) { - // Restart the track, which will cause Livekit to do another - // getUserMedia() call with deviceId: default to get the *new* default device. - // Note that room.switchActiveDevice() won't work: Livekit will ignore it because - // the deviceId hasn't changed (was & still is default). - this.livekitRoom.localParticipant - .getTrackPublication(Track.Source.Microphone) - ?.audioTrack?.restartTrack() - .catch((e) => { - logger.error(`Failed to restart audio device track`, e); - }); - } - }); - } -} diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts new file mode 100644 index 000000000..532be26c2 --- /dev/null +++ b/src/state/PublishConnection.ts @@ -0,0 +1,224 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ +import { ConnectionState, type E2EEOptions, LocalVideoTrack, Room as LivekitRoom, Track } from "livekit-client"; +import { map, NEVER, type Observable, type Subscription, switchMap } from "rxjs"; + +import type { CallMembership, LivekitFocus } from "../../../matrix-js-sdk/lib/matrixrtc"; +import type { MatrixClient } from "../../../matrix-js-sdk"; +import type { ObservableScope } from "./ObservableScope.ts"; +import type { Behavior } from "./Behavior.ts"; +import type { MediaDevices, SelectedDevice } from "./MediaDevices.ts"; +import type { MuteStates } from "./MuteStates.ts"; +import { type ProcessorState, trackProcessorSync } from "../livekit/TrackProcessorContext.tsx"; +import { logger } from "../../../matrix-js-sdk/lib/logger"; +import { getUrlParams } from "../UrlParams.ts"; +import { defaultLiveKitOptions } from "../livekit/options.ts"; +import { getValue } from "../utils/observable.ts"; +import { observeTrackReference$ } from "./MediaViewModel.ts"; +import { Connection } from "./Connection.ts"; + +/** + * A connection to the publishing LiveKit.e. the local livekit room, the one the user is publishing to. + * This connection will publish the local user's audio and video tracks. + */ +export class PublishConnection extends Connection { + + + /** + * Start the connection to LiveKit and publish local tracks. + * + * This will: + * 1. Request an OpenId token `request_token` (allows matrix users to verify their identity with a third-party service.) + * 2. Use this token to request the SFU config to the MatrixRtc authentication service. + * 3. Connect to the configured LiveKit room. + * 4. Create local audio and video tracks based on the current mute states and publish them to the room. + */ + public async start(): Promise { + this.stopped = false; + const { url, jwt } = await this.sfuConfig; + if (!this.stopped) await this.livekitRoom.connect(url, jwt); + + if (!this.stopped) { + // TODO this can throw errors? It will also prompt for permissions if not already granted + const tracks = await this.livekitRoom.localParticipant.createTracks({ + audio: this.muteStates.audio.enabled$.value, + video: this.muteStates.video.enabled$.value + }); + for (const track of tracks) { + // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally + // with a timeout. + await this.livekitRoom.localParticipant.publishTrack(track); + // TODO: check if the connection is still active? and break the loop if not? + } + } + }; + + + /** + * Creates a new PublishConnection. + * @param focus - The Livekit focus object containing the configuration for the connection. + * @param livekitAlias - TODO: remove, use focus.livekit_alias instead + * @param client - The Matrix client to use for authentication. TODO: remove only pick OpenIDClientParts + * @param scope - The observable scope to use for managing subscriptions. + * @param membershipsFocusMap$ - An observable of the current RTC call memberships and their associated focus. + * @param devices - The media devices to use for audio and video input. + * @param muteStates - The mute states for audio and video. + * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. + * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur). + */ + public constructor( + focus: LivekitFocus, + livekitAlias: string, + client: MatrixClient, + scope: ObservableScope, + membershipsFocusMap$: Behavior< + { membership: CallMembership; focus: LivekitFocus }[] + >, + devices: MediaDevices, + private readonly muteStates: MuteStates, + e2eeLivekitOptions: E2EEOptions | undefined, + trackerProcessorState$: Behavior + ) { + logger.info("[LivekitRoom] Create LiveKit room"); + const { controlledAudioDevices } = getUrlParams(); + + const room = new LivekitRoom({ + ...defaultLiveKitOptions, + videoCaptureDefaults: { + ...defaultLiveKitOptions.videoCaptureDefaults, + deviceId: devices.videoInput.selected$.value?.id, + processor: trackerProcessorState$.value.processor + }, + audioCaptureDefaults: { + ...defaultLiveKitOptions.audioCaptureDefaults, + deviceId: devices.audioInput.selected$.value?.id + }, + audioOutput: { + // When using controlled audio devices, we don't want to set the + // deviceId here, because it will be set by the native app. + // (also the id does not need to match a browser device id) + deviceId: controlledAudioDevices + ? undefined + : getValue(devices.audioOutput.selected$)?.id + }, + e2ee: e2eeLivekitOptions + }); + room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => { + logger.error("Failed to set E2EE enabled on room", e); + }); + + super( + focus, + livekitAlias, + client, + scope, + membershipsFocusMap$, + e2eeLivekitOptions, + room + ); + + // Setup track processor syncing (blur) + const track$ = this.scope.behavior( + observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( + map((trackRef) => { + const track = trackRef?.publication?.track; + return track instanceof LocalVideoTrack ? track : null; + }) + ) + ); + trackProcessorSync(track$, trackerProcessorState$); + + this.muteStates.audio.setHandler(async (desired) => { + try { + await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); + } catch (e) { + logger.error("Failed to update LiveKit audio input mute state", e); + } + return this.livekitRoom.localParticipant.isMicrophoneEnabled; + }); + this.muteStates.video.setHandler(async (desired) => { + try { + await this.livekitRoom.localParticipant.setCameraEnabled(desired); + } catch (e) { + logger.error("Failed to update LiveKit video input mute state", e); + } + return this.livekitRoom.localParticipant.isCameraEnabled; + }); + this.scope.onEnd(() => { + this.muteStates.audio.unsetHandler(); + this.muteStates.video.unsetHandler(); + }); + + const syncDevice = ( + kind: MediaDeviceKind, + selected$: Observable + ): Subscription => + selected$.pipe(this.scope.bind()).subscribe((device) => { + if (this.connectionState$.value !== ConnectionState.Connected) return; + logger.info( + "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", + this.livekitRoom.getActiveDevice(kind), + " !== ", + device?.id + ); + if ( + device !== undefined && + this.livekitRoom.getActiveDevice(kind) !== device.id + ) { + this.livekitRoom + .switchActiveDevice(kind, device.id) + .catch((e) => + logger.error(`Failed to sync ${kind} device with LiveKit`, e) + ); + } + }); + + syncDevice("audioinput", devices.audioInput.selected$); + if (!controlledAudioDevices) + syncDevice("audiooutput", devices.audioOutput.selected$); + syncDevice("videoinput", devices.videoInput.selected$); + // Restart the audio input track whenever we detect that the active media + // device has changed to refer to a different hardware device. We do this + // for the sake of Chrome, which provides a "default" device that is meant + // to match the system's default audio input, whatever that may be. + // This is special-cased for only audio inputs because we need to dig around + // in the LocalParticipant object for the track object and there's not a nice + // way to do that generically. There is usually no OS-level default video capture + // device anyway, and audio outputs work differently. + devices.audioInput.selected$ + .pipe( + switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), + this.scope.bind() + ) + .subscribe(() => { + if (this.connectionState$.value !== ConnectionState.Connected) return; + const activeMicTrack = Array.from( + this.livekitRoom.localParticipant.audioTrackPublications.values() + ).find((d) => d.source === Track.Source.Microphone)?.track; + + if ( + activeMicTrack && + // only restart if the stream is still running: LiveKit will detect + // when a track stops & restart appropriately, so this is not our job. + // Plus, we need to avoid restarting again if the track is already in + // the process of being restarted. + activeMicTrack.mediaStreamTrack.readyState !== "ended" + ) { + // Restart the track, which will cause Livekit to do another + // getUserMedia() call with deviceId: default to get the *new* default device. + // Note that room.switchActiveDevice() won't work: Livekit will ignore it because + // the deviceId hasn't changed (was & still is default). + this.livekitRoom.localParticipant + .getTrackPublication(Track.Source.Microphone) + ?.audioTrack?.restartTrack() + .catch((e) => { + logger.error(`Failed to restart audio device track`, e); + }); + } + }); + } +} From b00f7d54099c0d26b4bb865dd458662b8d9eb59e Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 30 Sep 2025 17:02:48 +0200 Subject: [PATCH 02/46] refactor: Remote / Publish Connection and constructor --- src/room/InCallView.tsx | 1 - src/state/CallViewModel.ts | 36 ++++++----- src/state/Connection.ts | 112 +++++++++++++++++++-------------- src/state/PublishConnection.ts | 48 ++++++-------- 4 files changed, 104 insertions(+), 93 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index db2c0f2af..57873b400 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -31,7 +31,6 @@ import { VolumeOnSolidIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import { useTranslation } from "react-i18next"; -import { ConnectionState } from "livekit-client"; import LogoMark from "../icons/LogoMark.svg?react"; import LogoType from "../icons/LogoType.svg?react"; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index b6327cfa0..6b2ee35af 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -122,7 +122,7 @@ import { } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { Connection } from "./Connection"; +import { type Connection, type ConnectionOpts, RemoteConnection } from "./Connection"; import { type MuteStates } from "./MuteStates"; import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; @@ -453,18 +453,21 @@ export class CallViewModel extends ViewModel { private readonly localFocus = makeFocus(this.matrixRTCSession); private readonly localConnection = this.localFocus.then( - (focus) => - new PublishConnection( + (focus) => { + const args: ConnectionOpts = { focus, - this.livekitAlias, - this.matrixRTCSession.room.client, - this.scope, - this.membershipsAndFocusMap$, + client: this.matrixRTCSession.room.client, + scope: this.scope, + membershipsFocusMap$: this.membershipsAndFocusMap$, + } + return new PublishConnection( + args, this.mediaDevices, this.muteStates, this.e2eeLivekitOptions(), this.scope.behavior(this.trackProcessorState$), - ), + ) + } ); public readonly livekitConnectionState$ = this.scope.behavior( @@ -521,18 +524,17 @@ export class CallViewModel extends ViewModel { "SFU remoteConnections$ construct new connection: ", focusUrl, ); - nextConnection = new Connection( - { + const args: ConnectionOpts = { + focus: { + type: "livekit", livekit_service_url: focusUrl, livekit_alias: this.livekitAlias, - type: "livekit", }, - this.livekitAlias, - this.matrixRTCSession.room.client, - this.scope, - this.membershipsAndFocusMap$, - this.e2eeLivekitOptions(), - ); + client: this.matrixRTCSession.room.client, + scope: this.scope, + membershipsFocusMap$: this.membershipsAndFocusMap$, + } + nextConnection = new RemoteConnection(args, this.e2eeLivekitOptions()); } else { logger.log( "SFU remoteConnections$ use prev connection: ", diff --git a/src/state/Connection.ts b/src/state/Connection.ts index f725ddda2..bc352adf1 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -7,15 +7,24 @@ Please see LICENSE in the repository root for full details. import { connectedParticipantsObserver, connectionStateObserver } from "@livekit/components-core"; import { type ConnectionState, type E2EEOptions, Room as LivekitRoom } from "livekit-client"; -import { type MatrixClient } from "matrix-js-sdk"; import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; import { combineLatest } from "rxjs"; -import { getSFUConfigWithOpenID } from "../livekit/openIDSFU"; +import { getSFUConfigWithOpenID, type OpenIDClientParts, type SFUConfig } from "../livekit/openIDSFU"; import { type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; import { defaultLiveKitOptions } from "../livekit/options"; +export interface ConnectionOpts { + /** The focus server to connect to. */ + focus: LivekitFocus; + /** The Matrix client to use for OpenID and SFU config requests. */ + client: OpenIDClientParts; + /** The observable scope to use for this connection. */ + scope: ObservableScope; + /** An observable of the current RTC call memberships and their associated focus. */ + membershipsFocusMap$: Behavior<{ membership: CallMembership; focus: LivekitFocus }[]>; +} /** * A connection to a Matrix RTC LiveKit backend. * @@ -39,10 +48,20 @@ export class Connection { */ public async start(): Promise { this.stopped = false; - const { url, jwt } = await this.sfuConfig; + // TODO could this be loaded earlier to save time? + const { url, jwt } = await this.getSFUConfigWithOpenID(); + if (!this.stopped) await this.livekitRoom.connect(url, jwt); } + + protected async getSFUConfigWithOpenID(): Promise { + return await getSFUConfigWithOpenID( + this.client, + this.targetFocus.livekit_service_url, + this.targetFocus.livekit_alias + ) + } /** * Stops the connection. * @@ -55,17 +74,6 @@ export class Connection { this.stopped = true; } - protected readonly sfuConfig = getSFUConfigWithOpenID( - this.client, - this.focus.livekit_service_url, - this.focus.livekit_alias - ); - - /* - * An observable of the participants in the livekit room, including subscribers. - * Converts the livekit room events ParticipantConnected/ParticipantDisconnected/StateChange to an observable. - */ - protected readonly participantsIncludingSubscribers$; /** * An observable of the participants that are publishing on this connection. @@ -75,9 +83,9 @@ export class Connection { public readonly publishingParticipants$; /** - * The LiveKit room instance. + * The focus server to connect to. */ - public readonly livekitRoom: LivekitRoom; + protected readonly targetFocus: LivekitFocus; /** * An observable of the livekit connection state. @@ -85,48 +93,39 @@ export class Connection { */ public connectionState$: Behavior; + + private readonly client: OpenIDClientParts; /** * Creates a new connection to a matrix RTC LiveKit backend. * - * @param livekitRoom - Optional LiveKit room instance to use. If not provided, a new instance will be created. - * @param focus - The focus server to connect to. - * @param livekitAlias - The livekit alias to use when connecting to the focus server. TODO duplicate of focus? - * @param client - The matrix client, used to fetch the OpenId token. TODO refactor to avoid passing the whole client - * @param scope - The observable scope to use for creating observables. - * @param membershipsFocusMap$ - The observable of the current call RTC memberships and their associated focus. - * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. TODO refactor to avoid passing the whole options? + * @param livekitRoom - LiveKit room instance to use. + * @param opts - Connection options {@link ConnectionOpts}. + * */ - public constructor( - protected readonly focus: LivekitFocus, - // TODO : remove livekitAlias, it's already in focus? - protected readonly livekitAlias: string, - protected readonly client: MatrixClient, - protected readonly scope: ObservableScope, - protected readonly membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitFocus }[] - >, - e2eeLivekitOptions: E2EEOptions | undefined, - livekitRoom: LivekitRoom | undefined = undefined + protected constructor( + public readonly livekitRoom: LivekitRoom, + opts: ConnectionOpts, ) { - this.livekitRoom = - livekitRoom ?? - new LivekitRoom({ - ...defaultLiveKitOptions, - e2ee: e2eeLivekitOptions - }); - this.participantsIncludingSubscribers$ = this.scope.behavior( + const { focus, client, scope, membershipsFocusMap$ } = + opts; + + this.livekitRoom = livekitRoom + this.targetFocus = focus; + this.client = client; + + const participantsIncludingSubscribers$ = scope.behavior( connectedParticipantsObserver(this.livekitRoom), [] ); - this.publishingParticipants$ = this.scope.behavior( + this.publishingParticipants$ = scope.behavior( combineLatest( - [this.participantsIncludingSubscribers$, this.membershipsFocusMap$], + [participantsIncludingSubscribers$, membershipsFocusMap$], (participants, membershipsFocusMap) => membershipsFocusMap // Find all members that claim to publish on this connection .flatMap(({ membership, focus }) => - focus.livekit_service_url === this.focus.livekit_service_url + focus.livekit_service_url === this.targetFocus.livekit_service_url ? [membership] : [] ) @@ -141,11 +140,32 @@ export class Connection { ), [] ); - this.connectionState$ = this.scope.behavior( + this.connectionState$ = scope.behavior( connectionStateObserver(this.livekitRoom) ); - this.scope.onEnd(() => this.stop()); + scope.onEnd(() => this.stop()); } } +/** + * A remote connection to the Matrix RTC LiveKit backend. + * + * This connection is used for subscribing to remote participants. + * It does not publish any local tracks. + */ +export class RemoteConnection extends Connection { + + /** + * Creates a new remote connection to a matrix RTC LiveKit backend. + * @param opts + * @param sharedE2eeOption - The shared E2EE options to use for the connection. + */ + public constructor(opts: ConnectionOpts, sharedE2eeOption: E2EEOptions | undefined) { + const livekitRoom = new LivekitRoom({ + ...defaultLiveKitOptions, + e2ee: sharedE2eeOption + }); + super(livekitRoom, opts); + } +} diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts index 532be26c2..724c6c5f2 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/PublishConnection.ts @@ -7,9 +7,6 @@ Please see LICENSE in the repository root for full details. import { ConnectionState, type E2EEOptions, LocalVideoTrack, Room as LivekitRoom, Track } from "livekit-client"; import { map, NEVER, type Observable, type Subscription, switchMap } from "rxjs"; -import type { CallMembership, LivekitFocus } from "../../../matrix-js-sdk/lib/matrixrtc"; -import type { MatrixClient } from "../../../matrix-js-sdk"; -import type { ObservableScope } from "./ObservableScope.ts"; import type { Behavior } from "./Behavior.ts"; import type { MediaDevices, SelectedDevice } from "./MediaDevices.ts"; import type { MuteStates } from "./MuteStates.ts"; @@ -19,7 +16,7 @@ import { getUrlParams } from "../UrlParams.ts"; import { defaultLiveKitOptions } from "../livekit/options.ts"; import { getValue } from "../utils/observable.ts"; import { observeTrackReference$ } from "./MediaViewModel.ts"; -import { Connection } from "./Connection.ts"; +import { Connection, type ConnectionOpts } from "./Connection.ts"; /** * A connection to the publishing LiveKit.e. the local livekit room, the one the user is publishing to. @@ -39,8 +36,8 @@ export class PublishConnection extends Connection { */ public async start(): Promise { this.stopped = false; - const { url, jwt } = await this.sfuConfig; - if (!this.stopped) await this.livekitRoom.connect(url, jwt); + + await super.start() if (!this.stopped) { // TODO this can throw errors? It will also prompt for permissions if not already granted @@ -60,29 +57,20 @@ export class PublishConnection extends Connection { /** * Creates a new PublishConnection. - * @param focus - The Livekit focus object containing the configuration for the connection. - * @param livekitAlias - TODO: remove, use focus.livekit_alias instead - * @param client - The Matrix client to use for authentication. TODO: remove only pick OpenIDClientParts - * @param scope - The observable scope to use for managing subscriptions. - * @param membershipsFocusMap$ - An observable of the current RTC call memberships and their associated focus. + * @param args - The connection options. {@link ConnectionOpts} * @param devices - The media devices to use for audio and video input. * @param muteStates - The mute states for audio and video. * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur). */ public constructor( - focus: LivekitFocus, - livekitAlias: string, - client: MatrixClient, - scope: ObservableScope, - membershipsFocusMap$: Behavior< - { membership: CallMembership; focus: LivekitFocus }[] - >, + args: ConnectionOpts, devices: MediaDevices, private readonly muteStates: MuteStates, e2eeLivekitOptions: E2EEOptions | undefined, trackerProcessorState$: Behavior ) { + const { scope } = args; logger.info("[LivekitRoom] Create LiveKit room"); const { controlledAudioDevices } = getUrlParams(); @@ -112,17 +100,19 @@ export class PublishConnection extends Connection { }); super( - focus, - livekitAlias, - client, - scope, - membershipsFocusMap$, - e2eeLivekitOptions, - room + room, + args, + // focus, + // livekitAlias, + // client, + // scope, + // membershipsFocusMap$, + // e2eeLivekitOptions, + // room ); // Setup track processor syncing (blur) - const track$ = this.scope.behavior( + const track$ = scope.behavior( observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( map((trackRef) => { const track = trackRef?.publication?.track; @@ -148,7 +138,7 @@ export class PublishConnection extends Connection { } return this.livekitRoom.localParticipant.isCameraEnabled; }); - this.scope.onEnd(() => { + scope.onEnd(() => { this.muteStates.audio.unsetHandler(); this.muteStates.video.unsetHandler(); }); @@ -157,7 +147,7 @@ export class PublishConnection extends Connection { kind: MediaDeviceKind, selected$: Observable ): Subscription => - selected$.pipe(this.scope.bind()).subscribe((device) => { + selected$.pipe(scope.bind()).subscribe((device) => { if (this.connectionState$.value !== ConnectionState.Connected) return; logger.info( "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", @@ -192,7 +182,7 @@ export class PublishConnection extends Connection { devices.audioInput.selected$ .pipe( switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), - this.scope.bind() + scope.bind() ) .subscribe(() => { if (this.connectionState$.value !== ConnectionState.Connected) return; From 879a1d4af1e96f1ff3b9df5c338f3ee14d881016 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 1 Oct 2025 10:06:43 +0200 Subject: [PATCH 03/46] Connection: add Connection state and handle error on start --- src/state/CallViewModel.ts | 22 ++++++++--- src/state/Connection.ts | 69 ++++++++++++++++++++++++++-------- src/state/PublishConnection.ts | 48 +++++++++++------------ 3 files changed, 90 insertions(+), 49 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 6b2ee35af..cac4322e2 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -470,12 +470,22 @@ export class CallViewModel extends ViewModel { } ); - public readonly livekitConnectionState$ = this.scope.behavior( - combineLatest([this.localConnection]).pipe( - switchMap(([c]) => c.connectionState$), - startWith(ConnectionState.Disconnected), - ), - ); + public readonly livekitConnectionState$ = + this.scope.behavior( + from(this.localConnection).pipe( + switchMap((c) => + c.focusedConnectionState$.pipe( + map((s) => { + if (s.state === "ConnectedToLkRoom") return s.connectionState; + return ConnectionState.Disconnected + }), + distinctUntilChanged(), + ), + ), + startWith(ConnectionState.Disconnected), + ), + ) + /** * The MatrixRTC session participants. diff --git a/src/state/Connection.ts b/src/state/Connection.ts index bc352adf1..1e081b067 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -6,9 +6,9 @@ Please see LICENSE in the repository root for full details. */ import { connectedParticipantsObserver, connectionStateObserver } from "@livekit/components-core"; -import { type ConnectionState, type E2EEOptions, Room as LivekitRoom } from "livekit-client"; +import { type ConnectionState, type E2EEOptions, Room as LivekitRoom, type RoomOptions } from "livekit-client"; import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; -import { combineLatest } from "rxjs"; +import { BehaviorSubject, combineLatest } from "rxjs"; import { getSFUConfigWithOpenID, type OpenIDClientParts, type SFUConfig } from "../livekit/openIDSFU"; import { type Behavior } from "./Behavior"; @@ -24,7 +24,20 @@ export interface ConnectionOpts { scope: ObservableScope; /** An observable of the current RTC call memberships and their associated focus. */ membershipsFocusMap$: Behavior<{ membership: CallMembership; focus: LivekitFocus }[]>; + + /** Optional factory to create the Livekit room, mainly for testing purposes. */ + livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; } + +export type FocusConnectionState = + | { state: 'Initialized' } + | { state: 'FetchingConfig', focus: LivekitFocus } + | { state: 'ConnectingToLkRoom', focus: LivekitFocus } + | { state: 'PublishingTracks', focus: LivekitFocus } + | { state: 'FailedToStart', error: Error, focus: LivekitFocus } + | { state: 'ConnectedToLkRoom', connectionState: ConnectionState, focus: LivekitFocus } + | { state: 'Stopped', focus: LivekitFocus }; + /** * A connection to a Matrix RTC LiveKit backend. * @@ -32,6 +45,15 @@ export interface ConnectionOpts { */ export class Connection { + // Private Behavior + private readonly _focusedConnectionState$ = new BehaviorSubject({ state: 'Initialized' }); + + /** + * The current state of the connection to the focus server. + */ + public get focusedConnectionState$(): Behavior { + return this._focusedConnectionState$; + } /** * Whether the connection has been stopped. * @see Connection.stop @@ -48,10 +70,23 @@ export class Connection { */ public async start(): Promise { this.stopped = false; - // TODO could this be loaded earlier to save time? - const { url, jwt } = await this.getSFUConfigWithOpenID(); - - if (!this.stopped) await this.livekitRoom.connect(url, jwt); + try { + this._focusedConnectionState$.next({ state: 'FetchingConfig', focus: this.targetFocus }); + // TODO could this be loaded earlier to save time? + const { url, jwt } = await this.getSFUConfigWithOpenID(); + // If we were stopped while fetching the config, don't proceed to connect + if (this.stopped) return; + + this._focusedConnectionState$.next({ state: 'ConnectingToLkRoom', focus: this.targetFocus }); + await this.livekitRoom.connect(url, jwt); + // If we were stopped while connecting, don't proceed to update state. + if (this.stopped) return; + + this._focusedConnectionState$.next({ state: 'ConnectedToLkRoom', focus: this.targetFocus, connectionState: this.livekitRoom.state }); + } catch (error) { + this._focusedConnectionState$.next({ state: 'FailedToStart', error: error instanceof Error ? error : new Error(`${error}`), focus: this.targetFocus }); + throw error; + } } @@ -71,6 +106,7 @@ export class Connection { public stop(): void { if (this.stopped) return; void this.livekitRoom.disconnect(); + this._focusedConnectionState$.next({ state: 'Stopped', focus: this.targetFocus }); this.stopped = true; } @@ -87,13 +123,6 @@ export class Connection { */ protected readonly targetFocus: LivekitFocus; - /** - * An observable of the livekit connection state. - * Converts the livekit room events StateChange to an observable. - */ - public connectionState$: Behavior; - - private readonly client: OpenIDClientParts; /** * Creates a new connection to a matrix RTC LiveKit backend. @@ -140,9 +169,16 @@ export class Connection { ), [] ); - this.connectionState$ = scope.behavior( + + scope.behavior( connectionStateObserver(this.livekitRoom) - ); + ).subscribe((connectionState) => { + const current = this.focusedConnectionState$.value; + // Only update the state if we are already connected to the LiveKit room. + if (current.state === 'ConnectedToLkRoom') { + this.focusedConnectionState$.next({ state: 'ConnectedToLkRoom', connectionState, focus: current.focus }); + } + }); scope.onEnd(() => this.stop()); } @@ -162,7 +198,8 @@ export class RemoteConnection extends Connection { * @param sharedE2eeOption - The shared E2EE options to use for the connection. */ public constructor(opts: ConnectionOpts, sharedE2eeOption: E2EEOptions | undefined) { - const livekitRoom = new LivekitRoom({ + const factory = opts.livekitRoomFactory ?? ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); + const livekitRoom = factory({ ...defaultLiveKitOptions, e2ee: sharedE2eeOption }); diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts index 724c6c5f2..c7b9c6aa9 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/PublishConnection.ts @@ -4,7 +4,7 @@ Copyright 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { ConnectionState, type E2EEOptions, LocalVideoTrack, Room as LivekitRoom, Track } from "livekit-client"; +import { ConnectionState, type E2EEOptions, LocalVideoTrack, Room as LivekitRoom, type RoomOptions, Track } from "livekit-client"; import { map, NEVER, type Observable, type Subscription, switchMap } from "rxjs"; import type { Behavior } from "./Behavior.ts"; @@ -39,18 +39,20 @@ export class PublishConnection extends Connection { await super.start() - if (!this.stopped) { - // TODO this can throw errors? It will also prompt for permissions if not already granted - const tracks = await this.livekitRoom.localParticipant.createTracks({ - audio: this.muteStates.audio.enabled$.value, - video: this.muteStates.video.enabled$.value - }); - for (const track of tracks) { - // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally - // with a timeout. - await this.livekitRoom.localParticipant.publishTrack(track); - // TODO: check if the connection is still active? and break the loop if not? - } + if (this.stopped) return; + + // TODO this can throw errors? It will also prompt for permissions if not already granted + const tracks = await this.livekitRoom.localParticipant.createTracks({ + audio: this.muteStates.audio.enabled$.value, + video: this.muteStates.video.enabled$.value + }); + if (this.stopped) return; + for (const track of tracks) { + // TODO: handle errors? Needs the signaling connection to be up, but it has some retries internally + // with a timeout. + await this.livekitRoom.localParticipant.publishTrack(track); + if (this.stopped) return; + // TODO: check if the connection is still active? and break the loop if not? } }; @@ -74,7 +76,8 @@ export class PublishConnection extends Connection { logger.info("[LivekitRoom] Create LiveKit room"); const { controlledAudioDevices } = getUrlParams(); - const room = new LivekitRoom({ + const factory = args.livekitRoomFactory ?? ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); + const room = factory({ ...defaultLiveKitOptions, videoCaptureDefaults: { ...defaultLiveKitOptions.videoCaptureDefaults, @@ -99,17 +102,7 @@ export class PublishConnection extends Connection { logger.error("Failed to set E2EE enabled on room", e); }); - super( - room, - args, - // focus, - // livekitAlias, - // client, - // scope, - // membershipsFocusMap$, - // e2eeLivekitOptions, - // room - ); + super(room, args); // Setup track processor syncing (blur) const track$ = scope.behavior( @@ -148,7 +141,8 @@ export class PublishConnection extends Connection { selected$: Observable ): Subscription => selected$.pipe(scope.bind()).subscribe((device) => { - if (this.connectionState$.value !== ConnectionState.Connected) return; + if (this.livekitRoom.state != ConnectionState.Connected) return; + // if (this.connectionState$.value !== ConnectionState.Connected) return; logger.info( "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", this.livekitRoom.getActiveDevice(kind), @@ -185,7 +179,7 @@ export class PublishConnection extends Connection { scope.bind() ) .subscribe(() => { - if (this.connectionState$.value !== ConnectionState.Connected) return; + if (this.livekitRoom.state != ConnectionState.Connected) return; const activeMicTrack = Array.from( this.livekitRoom.localParticipant.audioTrackPublications.values() ).find((d) => d.source === Track.Source.Microphone)?.track; From 3d8639df0331f5d9ffc66766646d5fbf9898713d Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 1 Oct 2025 14:21:37 +0200 Subject: [PATCH 04/46] Connection states tests --- package.json | 1 + src/state/Connection.test.ts | 310 +++++++++++++++++++++++++++++++++++ 2 files changed, 311 insertions(+) create mode 100644 src/state/Connection.test.ts diff --git a/package.json b/package.json index 915830233..ff3d98f64 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,7 @@ "eslint-plugin-react-hooks": "^5.0.0", "eslint-plugin-rxjs": "^5.0.3", "eslint-plugin-unicorn": "^56.0.0", + "fetch-mock": "11.1.5", "global-jsdom": "^26.0.0", "i18next": "^24.0.0", "i18next-browser-languagedetector": "^8.0.0", diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts new file mode 100644 index 000000000..2764a0e1b --- /dev/null +++ b/src/state/Connection.test.ts @@ -0,0 +1,310 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { afterEach, vi, it, describe, type MockedObject, expect } from "vitest"; +import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; +import { BehaviorSubject } from "rxjs"; +import { type Room as LivekitRoom } from "livekit-client"; +import fetchMock from "fetch-mock"; + +import { type ConnectionOpts, type FocusConnectionState, RemoteConnection } from "./Connection.ts"; +import { ObservableScope } from "./ObservableScope.ts"; +import { type OpenIDClientParts, type SFUConfig } from "../livekit/openIDSFU.ts"; +import { FailToGetOpenIdToken } from "../utils/errors.ts"; + +describe("Start connection states", () => { + + let testScope: ObservableScope; + + let client: MockedObject; + + let fakeLivekitRoom: MockedObject; + + let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>; + + const livekitFocus : LivekitFocus = { + livekit_alias:"!roomID:example.org", + livekit_service_url : "https://matrix-rtc.example.org/livekit/jwt" + } + + afterEach(() => { + vi.useRealTimers(); + vi.clearAllMocks(); + fetchMock.reset(); + }) + + function setupTest(): void { + testScope = new ObservableScope(); + client = vi.mocked({ + getOpenIdToken: vi.fn().mockResolvedValue( + { + "access_token": "rYsmGUEwNjKgJYyeNUkZseJN", + "token_type": "Bearer", + "matrix_server_name": "example.org", + "expires_in": 3600 + } + ), + getDeviceId: vi.fn().mockReturnValue("ABCDEF"), + } as unknown as OpenIDClientParts); + fakeMembershipsFocusMap$ = new BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>([]); + + fakeLivekitRoom = vi.mocked({ + connect: vi.fn(), + disconnect: vi.fn(), + remoteParticipants: new Map(), + on: vi.fn(), + off: vi.fn(), + addListener: vi.fn(), + removeListener: vi.fn(), + removeAllListeners: vi.fn(), + } as unknown as LivekitRoom); + + } + + it("start in initialized state", () => { + setupTest(); + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: () => fakeLivekitRoom, + } + const connection = new RemoteConnection( + opts, + undefined, + ); + + expect(connection.focusedConnectionState$.getValue().state) + .toEqual("Initialized"); + }); + + it("fail to getOpenId token then error state", async () => { + setupTest(); + vi.useFakeTimers(); + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: () => fakeLivekitRoom, + } + + + const connection = new RemoteConnection( + opts, + undefined, + ); + + let capturedState: FocusConnectionState | undefined = undefined; + connection.focusedConnectionState$.subscribe((value) => { + capturedState = value; + }); + + + const deferred = Promise.withResolvers(); + + client.getOpenIdToken.mockImplementation(async () => { + await deferred.promise; + }) + + connection.start() + .catch(() => { + // expected to throw + }) + + expect(capturedState.state).toEqual("FetchingConfig"); + + deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token"))); + + await vi.runAllTimersAsync(); + + if (capturedState.state === "FailedToStart") { + expect(capturedState.error.message).toEqual("Something went wrong"); + expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + } else { + expect.fail("Expected FailedToStart state but got " + capturedState.state); + } + + }); + + it("fail to get JWT token and error state", async () => { + setupTest(); + vi.useFakeTimers(); + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: () => fakeLivekitRoom, + } + + const connection = new RemoteConnection( + opts, + undefined, + ); + + let capturedState: FocusConnectionState | undefined = undefined; + connection.focusedConnectionState$.subscribe((value) => { + capturedState = value; + }); + + const deferredSFU = Promise.withResolvers(); + // mock the /sfu/get call + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, + async () => { + await deferredSFU.promise; + return { + status: 500, + body: "Internal Server Error", + } + } + ); + + + connection.start() + .catch(() => { + // expected to throw + }) + + expect(capturedState.state).toEqual("FetchingConfig"); + + deferredSFU.resolve(); + await vi.runAllTimersAsync(); + + if (capturedState.state === "FailedToStart") { + expect(capturedState.error.message).toContain("SFU Config fetch failed with exception Error"); + expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + } else { + expect.fail("Expected FailedToStart state but got " + capturedState.state); + } + + }); + + + it("fail to connect to livekit error state", async () => { + setupTest(); + vi.useFakeTimers(); + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: () => fakeLivekitRoom, + } + + const connection = new RemoteConnection( + opts, + undefined, + ); + + let capturedState: FocusConnectionState | undefined = undefined; + connection.focusedConnectionState$.subscribe((value) => { + capturedState = value; + }); + + + const deferredSFU = Promise.withResolvers(); + // mock the /sfu/get call + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, + () => { + return { + status: 200, + body: + { + "url": "wss://matrix-rtc.m.localhost/livekit/sfu", + "jwt": "ATOKEN", + }, + } + } + ); + + fakeLivekitRoom + .connect + .mockImplementation(async () => { + await deferredSFU.promise; + throw new Error("Failed to connect to livekit"); + }); + + connection.start() + .catch(() => { + // expected to throw + }) + + expect(capturedState.state).toEqual("FetchingConfig"); + + deferredSFU.resolve(); + await vi.runAllTimersAsync(); + + if (capturedState.state === "FailedToStart") { + expect(capturedState.error.message).toContain("Failed to connect to livekit"); + expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + } else { + expect.fail("Expected FailedToStart state but got " + capturedState.state); + } + + }); + + it("connection states happy path", async () => { + setupTest(); + vi.useFakeTimers(); + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: () => fakeLivekitRoom, + } + + const connection = new RemoteConnection( + opts, + undefined, + ); + + let capturedState: FocusConnectionState[] = []; + connection.focusedConnectionState$.subscribe((value) => { + capturedState.push(value); + }); + + // mock the /sfu/get call + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, + () => { + return { + status: 200, + body: + { + "url": "wss://matrix-rtc.m.localhost/livekit/sfu", + "jwt": "ATOKEN", + }, + } + } + ); + + fakeLivekitRoom + .connect + .mockResolvedValue(undefined); + + await connection.start(); + await vi.runAllTimersAsync(); + + let initialState = capturedState.shift(); + expect(initialState?.state).toEqual("Initialized"); + let fetchingState = capturedState.shift(); + expect(fetchingState?.state).toEqual("FetchingConfig"); + let connectingState = capturedState.shift(); + expect(connectingState?.state).toEqual("ConnectingToLkRoom"); + let connectedState = capturedState.shift(); + expect(connectedState?.state).toEqual("ConnectedToLkRoom"); + + }); + +}) From 47c876f3dfccd0b7bf840799b63869b350afba16 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 1 Oct 2025 14:37:03 +0200 Subject: [PATCH 05/46] lint fixes --- src/state/Connection.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 2764a0e1b..59c60c2e9 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -270,7 +270,7 @@ describe("Start connection states", () => { undefined, ); - let capturedState: FocusConnectionState[] = []; + const capturedState: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { capturedState.push(value); }); @@ -296,13 +296,13 @@ describe("Start connection states", () => { await connection.start(); await vi.runAllTimersAsync(); - let initialState = capturedState.shift(); + const initialState = capturedState.shift(); expect(initialState?.state).toEqual("Initialized"); - let fetchingState = capturedState.shift(); + const fetchingState = capturedState.shift(); expect(fetchingState?.state).toEqual("FetchingConfig"); - let connectingState = capturedState.shift(); + const connectingState = capturedState.shift(); expect(connectingState?.state).toEqual("ConnectingToLkRoom"); - let connectedState = capturedState.shift(); + const connectedState = capturedState.shift(); expect(connectedState?.state).toEqual("ConnectedToLkRoom"); }); From 22900161d62d7b854dbf7f1314ea790b6431738a Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 1 Oct 2025 14:47:45 +0200 Subject: [PATCH 06/46] extract common test setup --- src/state/Connection.test.ts | 68 ++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 59c60c2e9..9fe415c1f 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -65,6 +65,42 @@ describe("Start connection states", () => { } + async function setupRemoteConnection(): RemoteConnection { + + setupTest() + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: () => fakeLivekitRoom, + } + + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, + () => { + return { + status: 200, + body: + { + "url": "wss://matrix-rtc.m.localhost/livekit/sfu", + "jwt": "ATOKEN", + }, + } + } + ); + + fakeLivekitRoom + .connect + .mockResolvedValue(undefined); + + const connection = new RemoteConnection( + opts, + undefined, + ); + return connection; + } + it("start in initialized state", () => { setupTest(); @@ -254,45 +290,15 @@ describe("Start connection states", () => { }); it("connection states happy path", async () => { - setupTest(); vi.useFakeTimers(); - const opts: ConnectionOpts = { - client: client, - focus: livekitFocus, - membershipsFocusMap$: fakeMembershipsFocusMap$, - scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom, - } - - const connection = new RemoteConnection( - opts, - undefined, - ); + const connection = setupRemoteConnection(); const capturedState: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { capturedState.push(value); }); - // mock the /sfu/get call - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, - () => { - return { - status: 200, - body: - { - "url": "wss://matrix-rtc.m.localhost/livekit/sfu", - "jwt": "ATOKEN", - }, - } - } - ); - - fakeLivekitRoom - .connect - .mockResolvedValue(undefined); - await connection.start(); await vi.runAllTimersAsync(); From 6a1f7dd057e4f279570eb91bb1c76521472ea7d4 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 1 Oct 2025 15:23:24 +0200 Subject: [PATCH 07/46] ConnectionState: test livekit connection states --- src/state/Connection.test.ts | 65 +++++++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 9fe415c1f..15c5d88ee 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -8,8 +8,9 @@ Please see LICENSE in the repository root for full details. import { afterEach, vi, it, describe, type MockedObject, expect } from "vitest"; import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject } from "rxjs"; -import { type Room as LivekitRoom } from "livekit-client"; +import { type Room as LivekitRoom, RoomEvent, type RoomEventCallbacks, ConnectionState } from "livekit-client"; import fetchMock from "fetch-mock"; +import EventEmitter from "events"; import { type ConnectionOpts, type FocusConnectionState, RemoteConnection } from "./Connection.ts"; import { ObservableScope } from "./ObservableScope.ts"; @@ -24,6 +25,7 @@ describe("Start connection states", () => { let fakeLivekitRoom: MockedObject; + let fakeRoomEventEmiter: EventEmitter; let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>; const livekitFocus : LivekitFocus = { @@ -52,22 +54,23 @@ describe("Start connection states", () => { } as unknown as OpenIDClientParts); fakeMembershipsFocusMap$ = new BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>([]); + fakeRoomEventEmiter = new EventEmitter(); + fakeLivekitRoom = vi.mocked({ connect: vi.fn(), disconnect: vi.fn(), remoteParticipants: new Map(), - on: vi.fn(), - off: vi.fn(), - addListener: vi.fn(), - removeListener: vi.fn(), - removeAllListeners: vi.fn(), + state: ConnectionState.Disconnected, + on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter), + off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), + addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), + removeListener: fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), + removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), } as unknown as LivekitRoom); } - async function setupRemoteConnection(): RemoteConnection { - - setupTest() + function setupRemoteConnection(): RemoteConnection { const opts: ConnectionOpts = { client: client, @@ -291,6 +294,7 @@ describe("Start connection states", () => { it("connection states happy path", async () => { vi.useFakeTimers(); + setupTest() const connection = setupRemoteConnection(); @@ -313,4 +317,47 @@ describe("Start connection states", () => { }); + it("should relay livekit events once connected", async () => { + vi.useFakeTimers(); + setupTest() + + const connection = setupRemoteConnection(); + + await connection.start(); + await vi.runAllTimersAsync(); + + const capturedState: FocusConnectionState[] = []; + connection.focusedConnectionState$.subscribe((value) => { + capturedState.push(value); + }); + + const states = [ + ConnectionState.Disconnected, + ConnectionState.Connecting, + ConnectionState.Connected, + ConnectionState.SignalReconnecting, + ConnectionState.Connecting, + ConnectionState.Connected, + ConnectionState.Reconnecting, + ] + for (const state of states) { + fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state); + await vi.runAllTimersAsync(); + } + + await vi.runAllTimersAsync(); + + for (const state of states) { + const s = capturedState.shift(); + expect(s?.state).toEqual("ConnectedToLkRoom"); + expect(s?.connectionState).toEqual(state); + + // should always have the focus info + expect(s?.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + expect(s?.focus.livekit_service_url).toEqual(livekitFocus.livekit_service_url); + } + + }); + + }) From e8bf817f881463a6c46c931589ac60b393e92786 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 1 Oct 2025 16:39:21 +0200 Subject: [PATCH 08/46] tests: end scope tests --- src/state/CallViewModel.ts | 2 +- src/state/Connection.test.ts | 65 +++++++++++++++++++++++++++--------- src/state/Connection.ts | 18 ++++++---- 3 files changed, 62 insertions(+), 23 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index cac4322e2..31a7e32dc 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -1890,7 +1890,7 @@ export class CallViewModel extends ViewModel { this.startConnection$ .pipe(this.scope.bind()) .subscribe((c) => void c.start()); - this.stopConnection$.pipe(this.scope.bind()).subscribe((c) => c.stop()); + this.stopConnection$.pipe(this.scope.bind()).subscribe((c) => void c.stop()); combineLatest([this.localFocus, this.join$]) .pipe(this.scope.bind()) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 15c5d88ee..8552ec24a 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -28,9 +28,9 @@ describe("Start connection states", () => { let fakeRoomEventEmiter: EventEmitter; let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>; - const livekitFocus : LivekitFocus = { - livekit_alias:"!roomID:example.org", - livekit_service_url : "https://matrix-rtc.example.org/livekit/jwt" + const livekitFocus: LivekitFocus = { + livekit_alias: "!roomID:example.org", + livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt" } afterEach(() => { @@ -98,8 +98,8 @@ describe("Start connection states", () => { .mockResolvedValue(undefined); const connection = new RemoteConnection( - opts, - undefined, + opts, + undefined, ); return connection; } @@ -115,8 +115,8 @@ describe("Start connection states", () => { livekitRoomFactory: () => fakeLivekitRoom, } const connection = new RemoteConnection( - opts, - undefined, + opts, + undefined, ); expect(connection.focusedConnectionState$.getValue().state) @@ -254,7 +254,7 @@ describe("Start connection states", () => { const deferredSFU = Promise.withResolvers(); // mock the /sfu/get call fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, - () => { + () => { return { status: 200, body: @@ -318,15 +318,13 @@ describe("Start connection states", () => { }); it("should relay livekit events once connected", async () => { - vi.useFakeTimers(); setupTest() const connection = setupRemoteConnection(); await connection.start(); - await vi.runAllTimersAsync(); - const capturedState: FocusConnectionState[] = []; + let capturedState: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { capturedState.push(value); }); @@ -342,11 +340,8 @@ describe("Start connection states", () => { ] for (const state of states) { fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state); - await vi.runAllTimersAsync(); } - await vi.runAllTimersAsync(); - for (const state of states) { const s = capturedState.shift(); expect(s?.state).toEqual("ConnectedToLkRoom"); @@ -357,7 +352,47 @@ describe("Start connection states", () => { expect(s?.focus.livekit_service_url).toEqual(livekitFocus.livekit_service_url); } + // If the state is not ConnectedToLkRoom, no events should be relayed anymore + await connection.stop(); + capturedState = []; + for (const state of states) { + fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state); + } + + expect(capturedState.length).toEqual(0); + }); -}) + it("shutting down the scope should stop the connection", async () => { + setupTest() + vi.useFakeTimers(); + + const connection = setupRemoteConnection(); + + let capturedState: FocusConnectionState[] = []; + connection.focusedConnectionState$.subscribe((value) => { + capturedState.push(value); + }); + + await connection.start(); + + const stopSpy = vi.spyOn(connection, "stop"); + testScope.end(); + + + expect(stopSpy).toHaveBeenCalled(); + expect(fakeLivekitRoom.disconnect).toHaveBeenCalled(); + + /// Ensures that focusedConnectionState$ is bound to the scope. + capturedState = []; + // the subscription should be closed, and no new state should be received + // @ts-expect-error: Accessing private field for testing purposes + connection._focusedConnectionState$.next({ state: "Initialized" }); + // @ts-expect-error: Accessing private field for testing purposes + connection._focusedConnectionState$.next({ state: "ConnectingToLkRoom" }); + + expect(capturedState.length).toEqual(0); + }); + +}); diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 1e081b067..1b93b5237 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -46,14 +46,14 @@ export type FocusConnectionState = export class Connection { // Private Behavior - private readonly _focusedConnectionState$ = new BehaviorSubject({ state: 'Initialized' }); + private readonly _focusedConnectionState$ + = new BehaviorSubject({ state: 'Initialized' }); /** * The current state of the connection to the focus server. */ - public get focusedConnectionState$(): Behavior { - return this._focusedConnectionState$; - } + public readonly focusedConnectionState$: Behavior; + /** * Whether the connection has been stopped. * @see Connection.stop @@ -103,9 +103,9 @@ export class Connection { * This will disconnect from the LiveKit room. * If the connection is already stopped, this is a no-op. */ - public stop(): void { + public async stop(): Promise { if (this.stopped) return; - void this.livekitRoom.disconnect(); + await this.livekitRoom.disconnect(); this._focusedConnectionState$.next({ state: 'Stopped', focus: this.targetFocus }); this.stopped = true; } @@ -142,6 +142,10 @@ export class Connection { this.targetFocus = focus; this.client = client; + this.focusedConnectionState$ = scope.behavior( + this._focusedConnectionState$, { state: 'Initialized' } + ); + const participantsIncludingSubscribers$ = scope.behavior( connectedParticipantsObserver(this.livekitRoom), [] @@ -180,7 +184,7 @@ export class Connection { } }); - scope.onEnd(() => this.stop()); + scope.onEnd(() => void this.stop()); } } From dfaa6a33f4fe4c01907d2f405d7aefde7fa58475 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 1 Oct 2025 17:24:19 +0200 Subject: [PATCH 09/46] fix lint errors --- src/state/Connection.test.ts | 215 ++++++++++++++++++----------------- src/state/Connection.ts | 4 +- 2 files changed, 115 insertions(+), 104 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 8552ec24a..692aee86f 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -5,104 +5,107 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { afterEach, vi, it, describe, type MockedObject, expect } from "vitest"; +import { afterEach, describe, expect, it, type MockedObject, vi } from "vitest"; import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject } from "rxjs"; -import { type Room as LivekitRoom, RoomEvent, type RoomEventCallbacks, ConnectionState } from "livekit-client"; +import { ConnectionState, type Room as LivekitRoom, RoomEvent } from "livekit-client"; import fetchMock from "fetch-mock"; import EventEmitter from "events"; +import { type IOpenIDToken } from "matrix-js-sdk"; import { type ConnectionOpts, type FocusConnectionState, RemoteConnection } from "./Connection.ts"; import { ObservableScope } from "./ObservableScope.ts"; -import { type OpenIDClientParts, type SFUConfig } from "../livekit/openIDSFU.ts"; +import { type OpenIDClientParts } from "../livekit/openIDSFU.ts"; import { FailToGetOpenIdToken } from "../utils/errors.ts"; -describe("Start connection states", () => { - let testScope: ObservableScope; +let testScope: ObservableScope; - let client: MockedObject; +let client: MockedObject; - let fakeLivekitRoom: MockedObject; +let fakeLivekitRoom: MockedObject; - let fakeRoomEventEmiter: EventEmitter; - let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>; +let fakeRoomEventEmiter: EventEmitter; +let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>; - const livekitFocus: LivekitFocus = { - livekit_alias: "!roomID:example.org", - livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt" - } - - afterEach(() => { - vi.useRealTimers(); - vi.clearAllMocks(); - fetchMock.reset(); - }) - - function setupTest(): void { - testScope = new ObservableScope(); - client = vi.mocked({ - getOpenIdToken: vi.fn().mockResolvedValue( - { - "access_token": "rYsmGUEwNjKgJYyeNUkZseJN", - "token_type": "Bearer", - "matrix_server_name": "example.org", - "expires_in": 3600 - } - ), - getDeviceId: vi.fn().mockReturnValue("ABCDEF"), - } as unknown as OpenIDClientParts); - fakeMembershipsFocusMap$ = new BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>([]); - - fakeRoomEventEmiter = new EventEmitter(); - - fakeLivekitRoom = vi.mocked({ - connect: vi.fn(), - disconnect: vi.fn(), - remoteParticipants: new Map(), - state: ConnectionState.Disconnected, - on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter), - off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), - addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), - removeListener: fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), - removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), - } as unknown as LivekitRoom); +const livekitFocus: LivekitFocus = { + livekit_alias: "!roomID:example.org", + livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", + type: "livekit", +} +function setupTest(): void { + testScope = new ObservableScope(); + client = vi.mocked({ + getOpenIdToken: vi.fn().mockResolvedValue( + { + "access_token": "rYsmGUEwNjKgJYyeNUkZseJN", + "token_type": "Bearer", + "matrix_server_name": "example.org", + "expires_in": 3600 + } + ), + getDeviceId: vi.fn().mockReturnValue("ABCDEF"), + } as unknown as OpenIDClientParts); + fakeMembershipsFocusMap$ = new BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>([]); + + fakeRoomEventEmiter = new EventEmitter(); + + fakeLivekitRoom = vi.mocked({ + connect: vi.fn(), + disconnect: vi.fn(), + remoteParticipants: new Map(), + state: ConnectionState.Disconnected, + on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter), + off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), + addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), + removeListener: fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), + removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), + } as unknown as LivekitRoom); + +} + +function setupRemoteConnection(): RemoteConnection { + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: () => fakeLivekitRoom, } - function setupRemoteConnection(): RemoteConnection { - - const opts: ConnectionOpts = { - client: client, - focus: livekitFocus, - membershipsFocusMap$: fakeMembershipsFocusMap$, - scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom, + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, + () => { + return { + status: 200, + body: + { + "url": "wss://matrix-rtc.m.localhost/livekit/sfu", + "jwt": "ATOKEN", + }, + } } + ); - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, - () => { - return { - status: 200, - body: - { - "url": "wss://matrix-rtc.m.localhost/livekit/sfu", - "jwt": "ATOKEN", - }, - } - } - ); + fakeLivekitRoom + .connect + .mockResolvedValue(undefined); - fakeLivekitRoom - .connect - .mockResolvedValue(undefined); + return new RemoteConnection( + opts, + undefined, + ); +} - const connection = new RemoteConnection( - opts, - undefined, - ); - return connection; - } + +describe("Start connection states", () => { + + afterEach(() => { + vi.useRealTimers(); + vi.clearAllMocks(); + fetchMock.reset(); + }) it("start in initialized state", () => { setupTest(); @@ -141,16 +144,16 @@ describe("Start connection states", () => { undefined, ); - let capturedState: FocusConnectionState | undefined = undefined; + const capturedStates: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { - capturedState = value; + capturedStates.push(value); }); - const deferred = Promise.withResolvers(); + const deferred = Promise.withResolvers(); - client.getOpenIdToken.mockImplementation(async () => { - await deferred.promise; + client.getOpenIdToken.mockImplementation(async (): Promise => { + return await deferred.promise; }) connection.start() @@ -158,17 +161,19 @@ describe("Start connection states", () => { // expected to throw }) - expect(capturedState.state).toEqual("FetchingConfig"); + const capturedState = capturedStates.shift(); + expect(capturedState).toBeDefined(); + expect(capturedState!.state).toEqual("FetchingConfig"); deferred.reject(new FailToGetOpenIdToken(new Error("Failed to get token"))); await vi.runAllTimersAsync(); - if (capturedState.state === "FailedToStart") { - expect(capturedState.error.message).toEqual("Something went wrong"); - expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + if (capturedState!.state === "FailedToStart") { + expect(capturedState!.error.message).toEqual("Something went wrong"); + expect(capturedState!.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); } else { - expect.fail("Expected FailedToStart state but got " + capturedState.state); + expect.fail("Expected FailedToStart state but got " + capturedState?.state); } }); @@ -190,9 +195,9 @@ describe("Start connection states", () => { undefined, ); - let capturedState: FocusConnectionState | undefined = undefined; + const capturedStates: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { - capturedState = value; + capturedStates.push(value); }); const deferredSFU = Promise.withResolvers(); @@ -213,16 +218,18 @@ describe("Start connection states", () => { // expected to throw }) - expect(capturedState.state).toEqual("FetchingConfig"); + const capturedState = capturedStates.shift(); + expect(capturedState).toBeDefined() + expect(capturedState?.state).toEqual("FetchingConfig"); deferredSFU.resolve(); await vi.runAllTimersAsync(); - if (capturedState.state === "FailedToStart") { - expect(capturedState.error.message).toContain("SFU Config fetch failed with exception Error"); - expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + if (capturedState?.state === "FailedToStart") { + expect(capturedState?.error.message).toContain("SFU Config fetch failed with exception Error"); + expect(capturedState?.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); } else { - expect.fail("Expected FailedToStart state but got " + capturedState.state); + expect.fail("Expected FailedToStart state but got " + capturedState?.state); } }); @@ -245,9 +252,9 @@ describe("Start connection states", () => { undefined, ); - let capturedState: FocusConnectionState | undefined = undefined; + const capturedStates: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { - capturedState = value; + capturedStates.push(value) }); @@ -278,16 +285,19 @@ describe("Start connection states", () => { // expected to throw }) - expect(capturedState.state).toEqual("FetchingConfig"); + const capturedState = capturedStates.shift(); + expect(capturedState).toBeDefined() + + expect(capturedState?.state).toEqual("FetchingConfig"); deferredSFU.resolve(); await vi.runAllTimersAsync(); - if (capturedState.state === "FailedToStart") { + if (capturedState && capturedState?.state === "FailedToStart") { expect(capturedState.error.message).toContain("Failed to connect to livekit"); expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); } else { - expect.fail("Expected FailedToStart state but got " + capturedState.state); + expect.fail("Expected FailedToStart state but got " + JSON.stringify(capturedState)); } }); @@ -345,11 +355,12 @@ describe("Start connection states", () => { for (const state of states) { const s = capturedState.shift(); expect(s?.state).toEqual("ConnectedToLkRoom"); - expect(s?.connectionState).toEqual(state); + const connectedState = s as FocusConnectionState & { state: "ConnectedToLkRoom" }; + expect(connectedState.connectionState).toEqual(state); // should always have the focus info - expect(s?.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); - expect(s?.focus.livekit_service_url).toEqual(livekitFocus.livekit_service_url); + expect(connectedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + expect(connectedState.focus.livekit_service_url).toEqual(livekitFocus.livekit_service_url); } // If the state is not ConnectedToLkRoom, no events should be relayed anymore diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 1b93b5237..16dd26074 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -177,10 +177,10 @@ export class Connection { scope.behavior( connectionStateObserver(this.livekitRoom) ).subscribe((connectionState) => { - const current = this.focusedConnectionState$.value; + const current = this._focusedConnectionState$.value; // Only update the state if we are already connected to the LiveKit room. if (current.state === 'ConnectedToLkRoom') { - this.focusedConnectionState$.next({ state: 'ConnectedToLkRoom', connectionState, focus: current.focus }); + this._focusedConnectionState$.next({ state: 'ConnectedToLkRoom', connectionState, focus: current.focus }); } }); From 0502f66e21ac31ce07e5511f96aca3fab676c7fa Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 2 Oct 2025 12:53:59 +0200 Subject: [PATCH 10/46] tests: Add publisher observable tests --- src/state/Connection.test.ts | 203 +++++++++++++++++++++++++++-------- 1 file changed, 159 insertions(+), 44 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 692aee86f..5c725e833 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -8,7 +8,7 @@ Please see LICENSE in the repository root for full details. import { afterEach, describe, expect, it, type MockedObject, vi } from "vitest"; import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject } from "rxjs"; -import { ConnectionState, type Room as LivekitRoom, RoomEvent } from "livekit-client"; +import { ConnectionState, type RemoteParticipant, type Room as LivekitRoom, RoomEvent } from "livekit-client"; import fetchMock from "fetch-mock"; import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; @@ -31,8 +31,8 @@ let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; focu const livekitFocus: LivekitFocus = { livekit_alias: "!roomID:example.org", livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", - type: "livekit", -} + type: "livekit" +}; function setupTest(): void { testScope = new ObservableScope(); @@ -45,7 +45,7 @@ function setupTest(): void { "expires_in": 3600 } ), - getDeviceId: vi.fn().mockReturnValue("ABCDEF"), + getDeviceId: vi.fn().mockReturnValue("ABCDEF") } as unknown as OpenIDClientParts); fakeMembershipsFocusMap$ = new BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>([]); @@ -60,7 +60,7 @@ function setupTest(): void { off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), removeListener: fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), - removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), + removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter) } as unknown as LivekitRoom); } @@ -72,8 +72,8 @@ function setupRemoteConnection(): RemoteConnection { focus: livekitFocus, membershipsFocusMap$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom, - } + livekitRoomFactory: () => fakeLivekitRoom + }; fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { @@ -82,9 +82,9 @@ function setupRemoteConnection(): RemoteConnection { body: { "url": "wss://matrix-rtc.m.localhost/livekit/sfu", - "jwt": "ATOKEN", - }, - } + "jwt": "ATOKEN" + } + }; } ); @@ -94,7 +94,7 @@ function setupRemoteConnection(): RemoteConnection { return new RemoteConnection( opts, - undefined, + undefined ); } @@ -105,7 +105,7 @@ describe("Start connection states", () => { vi.useRealTimers(); vi.clearAllMocks(); fetchMock.reset(); - }) + }); it("start in initialized state", () => { setupTest(); @@ -115,11 +115,11 @@ describe("Start connection states", () => { focus: livekitFocus, membershipsFocusMap$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom, - } + livekitRoomFactory: () => fakeLivekitRoom + }; const connection = new RemoteConnection( opts, - undefined, + undefined ); expect(connection.focusedConnectionState$.getValue().state) @@ -135,13 +135,13 @@ describe("Start connection states", () => { focus: livekitFocus, membershipsFocusMap$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom, - } + livekitRoomFactory: () => fakeLivekitRoom + }; const connection = new RemoteConnection( opts, - undefined, + undefined ); const capturedStates: FocusConnectionState[] = []; @@ -154,14 +154,14 @@ describe("Start connection states", () => { client.getOpenIdToken.mockImplementation(async (): Promise => { return await deferred.promise; - }) + }); connection.start() .catch(() => { // expected to throw - }) + }); - const capturedState = capturedStates.shift(); + let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); expect(capturedState!.state).toEqual("FetchingConfig"); @@ -169,6 +169,7 @@ describe("Start connection states", () => { await vi.runAllTimersAsync(); + capturedState = capturedStates.pop(); if (capturedState!.state === "FailedToStart") { expect(capturedState!.error.message).toEqual("Something went wrong"); expect(capturedState!.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); @@ -187,12 +188,12 @@ describe("Start connection states", () => { focus: livekitFocus, membershipsFocusMap$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom, - } + livekitRoomFactory: () => fakeLivekitRoom + }; const connection = new RemoteConnection( opts, - undefined, + undefined ); const capturedStates: FocusConnectionState[] = []; @@ -207,8 +208,8 @@ describe("Start connection states", () => { await deferredSFU.promise; return { status: 500, - body: "Internal Server Error", - } + body: "Internal Server Error" + }; } ); @@ -216,15 +217,17 @@ describe("Start connection states", () => { connection.start() .catch(() => { // expected to throw - }) + }); - const capturedState = capturedStates.shift(); - expect(capturedState).toBeDefined() + let capturedState = capturedStates.pop(); + expect(capturedState).toBeDefined(); expect(capturedState?.state).toEqual("FetchingConfig"); deferredSFU.resolve(); await vi.runAllTimersAsync(); + capturedState = capturedStates.pop(); + if (capturedState?.state === "FailedToStart") { expect(capturedState?.error.message).toContain("SFU Config fetch failed with exception Error"); expect(capturedState?.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); @@ -244,17 +247,17 @@ describe("Start connection states", () => { focus: livekitFocus, membershipsFocusMap$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom, - } + livekitRoomFactory: () => fakeLivekitRoom + }; const connection = new RemoteConnection( opts, - undefined, + undefined ); const capturedStates: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { - capturedStates.push(value) + capturedStates.push(value); }); @@ -267,9 +270,9 @@ describe("Start connection states", () => { body: { "url": "wss://matrix-rtc.m.localhost/livekit/sfu", - "jwt": "ATOKEN", - }, - } + "jwt": "ATOKEN" + } + }; } ); @@ -283,16 +286,18 @@ describe("Start connection states", () => { connection.start() .catch(() => { // expected to throw - }) + }); - const capturedState = capturedStates.shift(); - expect(capturedState).toBeDefined() + let capturedState = capturedStates.pop(); + expect(capturedState).toBeDefined(); expect(capturedState?.state).toEqual("FetchingConfig"); deferredSFU.resolve(); await vi.runAllTimersAsync(); + capturedState = capturedStates.pop(); + if (capturedState && capturedState?.state === "FailedToStart") { expect(capturedState.error.message).toContain("Failed to connect to livekit"); expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); @@ -304,7 +309,7 @@ describe("Start connection states", () => { it("connection states happy path", async () => { vi.useFakeTimers(); - setupTest() + setupTest(); const connection = setupRemoteConnection(); @@ -328,7 +333,7 @@ describe("Start connection states", () => { }); it("should relay livekit events once connected", async () => { - setupTest() + setupTest(); const connection = setupRemoteConnection(); @@ -346,8 +351,8 @@ describe("Start connection states", () => { ConnectionState.SignalReconnecting, ConnectionState.Connecting, ConnectionState.Connected, - ConnectionState.Reconnecting, - ] + ConnectionState.Reconnecting + ]; for (const state of states) { fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state); } @@ -376,7 +381,7 @@ describe("Start connection states", () => { it("shutting down the scope should stop the connection", async () => { - setupTest() + setupTest(); vi.useFakeTimers(); const connection = setupRemoteConnection(); @@ -407,3 +412,113 @@ describe("Start connection states", () => { }); }); + + +function fakeRemoteLivekitParticipant(id: string): RemoteParticipant { + return vi.mocked({ + identity: id + } as unknown as RemoteParticipant); +} + +function fakeRtcMemberShip(userId: string, deviceId: string): CallMembership { + return vi.mocked({ + sender: userId, + deviceId: deviceId, + } as unknown as CallMembership); +} + +describe("Publishing participants observations", () => { + + + it("should emit the list of publishing participants", async () => { + setupTest(); + + const connection = setupRemoteConnection(); + + const bobIsAPublisher = Promise.withResolvers(); + const danIsAPublisher = Promise.withResolvers(); + const observedPublishers: { participant: RemoteParticipant; membership: CallMembership }[][] = []; + connection.publishingParticipants$.subscribe((publishers) => { + observedPublishers.push(publishers); + if (publishers.some((p) => p.participant.identity === "@bob:example.org:DEV111")) { + bobIsAPublisher.resolve(); + } + if (publishers.some((p) => p.participant.identity === "@dan:example.org:DEV333")) { + danIsAPublisher.resolve(); + } + }); + // The publishingParticipants$ observable is derived from the current members of the + // livekitRoom and the rtc membership in order to publish the members that are publishing + // on this connection. + + let participants: RemoteParticipant[]= [ + fakeRemoteLivekitParticipant("@alice:example.org:DEV000"), + fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + fakeRemoteLivekitParticipant("@carol:example.org:DEV222"), + fakeRemoteLivekitParticipant("@dan:example.org:DEV333") + ]; + + // Let's simulate 3 members on the livekitRoom + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") + .mockReturnValue( + new Map(participants.map((p) => [p.identity, p])) + ); + + for (const participant of participants) { + fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); + } + + // At this point there should be no publishers + expect(observedPublishers.pop()!.length).toEqual(0); + + + const otherFocus: LivekitFocus = { + livekit_alias: "!roomID:example.org", + livekit_service_url: "https://other-matrix-rtc.example.org/livekit/jwt", + type: "livekit" + } + + + const rtcMemberships = [ + // Say bob is on the same focus + { membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), focus: livekitFocus }, + // Alice and carol is on a different focus + { membership: fakeRtcMemberShip("@alice:example.org", "DEV000"), focus: otherFocus }, + { membership: fakeRtcMemberShip("@carol:example.org", "DEV222"), focus: otherFocus }, + // NO DAVE YET + ]; + // signal this change in rtc memberships + fakeMembershipsFocusMap$.next(rtcMemberships); + + // We should have bob has a publisher now + await bobIsAPublisher.promise; + const publishers = observedPublishers.pop(); + expect(publishers?.length).toEqual(1); + expect(publishers?.[0].participant.identity).toEqual("@bob:example.org:DEV111"); + + // Now let's make dan join the rtc memberships + rtcMemberships + .push({ membership: fakeRtcMemberShip("@dan:example.org", "DEV333"), focus: livekitFocus }); + fakeMembershipsFocusMap$.next(rtcMemberships); + + // We should have bob and dan has publishers now + await danIsAPublisher.promise; + const twoPublishers = observedPublishers.pop(); + expect(twoPublishers?.length).toEqual(2); + expect(twoPublishers?.some((p) => p.participant.identity === "@bob:example.org:DEV111")).toBeTruthy(); + expect(twoPublishers?.some((p) => p.participant.identity === "@dan:example.org:DEV333")).toBeTruthy(); + + // Now let's make bob leave the livekit room + participants = participants.filter((p) => p.identity !== "@bob:example.org:DEV111"); + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") + .mockReturnValue( + new Map(participants.map((p) => [p.identity, p])) + ); + fakeRoomEventEmiter.emit(RoomEvent.ParticipantDisconnected, fakeRemoteLivekitParticipant("@bob:example.org:DEV111")); + + const updatedPublishers = observedPublishers.pop(); + expect(updatedPublishers?.length).toEqual(1); + expect(updatedPublishers?.some((p) => p.participant.identity === "@dan:example.org:DEV333")).toBeTruthy(); + }) + +}); From 84f95be48d8715eefa48b85deb89ada4f1da0889 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 2 Oct 2025 13:08:00 +0200 Subject: [PATCH 11/46] test: Ensure scope for publishers observer --- src/state/Connection.test.ts | 69 ++++++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 5c725e833..5529e588b 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -99,13 +99,14 @@ function setupRemoteConnection(): RemoteConnection { } -describe("Start connection states", () => { +afterEach(() => { + vi.useRealTimers(); + vi.clearAllMocks(); + fetchMock.reset(); +}); - afterEach(() => { - vi.useRealTimers(); - vi.clearAllMocks(); - fetchMock.reset(); - }); + +describe("Start connection states", () => { it("start in initialized state", () => { setupTest(); @@ -521,4 +522,60 @@ describe("Publishing participants observations", () => { expect(updatedPublishers?.some((p) => p.participant.identity === "@dan:example.org:DEV333")).toBeTruthy(); }) + + it("should be scoped to parent scope", async () => { + setupTest(); + + const connection = setupRemoteConnection(); + + let observedPublishers: { participant: RemoteParticipant; membership: CallMembership }[][] = []; + connection.publishingParticipants$.subscribe((publishers) => { + observedPublishers.push(publishers); + }); + + let participants: RemoteParticipant[]= [ + fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + ]; + + // Let's simulate 3 members on the livekitRoom + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") + .mockReturnValue( + new Map(participants.map((p) => [p.identity, p])) + ); + + for (const participant of participants) { + fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); + } + + // At this point there should be no publishers + expect(observedPublishers.pop()!.length).toEqual(0); + + const rtcMemberships = [ + // Say bob is on the same focus + { membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), focus: livekitFocus }, + ]; + // signal this change in rtc memberships + fakeMembershipsFocusMap$.next(rtcMemberships); + + // We should have bob has a publisher now + const publishers = observedPublishers.pop(); + expect(publishers?.length).toEqual(1); + expect(publishers?.[0].participant.identity).toEqual("@bob:example.org:DEV111"); + + // end the parent scope + testScope.end(); + observedPublishers = []; + + // SHOULD NOT emit any more publishers as the scope is ended + participants = participants.filter((p) => p.identity !== "@bob:example.org:DEV111"); + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") + .mockReturnValue( + new Map(participants.map((p) => [p.identity, p])) + ); + fakeRoomEventEmiter.emit(RoomEvent.ParticipantDisconnected, fakeRemoteLivekitParticipant("@bob:example.org:DEV111")); + + expect(observedPublishers.length).toEqual(0); + }) + + }); From 00401ca38ab99aa0fd32469425768ffc7d99ffa5 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 2 Oct 2025 15:15:23 +0200 Subject: [PATCH 12/46] refactor: PublishConnection extract from giant constructor --- src/state/PublishConnection.ts | 274 +++++++++++++++++++-------------- 1 file changed, 160 insertions(+), 114 deletions(-) diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts index c7b9c6aa9..6c15fc0fe 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/PublishConnection.ts @@ -4,7 +4,14 @@ Copyright 2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { ConnectionState, type E2EEOptions, LocalVideoTrack, Room as LivekitRoom, type RoomOptions, Track } from "livekit-client"; +import { + ConnectionState, + type E2EEOptions, + LocalVideoTrack, + Room as LivekitRoom, + type RoomOptions, + Track +} from "livekit-client"; import { map, NEVER, type Observable, type Subscription, switchMap } from "rxjs"; import type { Behavior } from "./Behavior.ts"; @@ -17,6 +24,7 @@ import { defaultLiveKitOptions } from "../livekit/options.ts"; import { getValue } from "../utils/observable.ts"; import { observeTrackReference$ } from "./MediaViewModel.ts"; import { Connection, type ConnectionOpts } from "./Connection.ts"; +import { type ObservableScope } from "./ObservableScope.ts"; /** * A connection to the publishing LiveKit.e. the local livekit room, the one the user is publishing to. @@ -24,6 +32,44 @@ import { Connection, type ConnectionOpts } from "./Connection.ts"; */ export class PublishConnection extends Connection { + /** + * Creates a new PublishConnection. + * @param args - The connection options. {@link ConnectionOpts} + * @param devices - The media devices to use for audio and video input. + * @param muteStates - The mute states for audio and video. + * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. + * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur). + */ + public constructor( + args: ConnectionOpts, + devices: MediaDevices, + private readonly muteStates: MuteStates, + e2eeLivekitOptions: E2EEOptions | undefined, + trackerProcessorState$: Behavior + ) { + const { scope } = args; + logger.info("[LivekitRoom] Create LiveKit room"); + const { controlledAudioDevices } = getUrlParams(); + + const factory = args.livekitRoomFactory ?? ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); + const room = factory( + generateRoomOption(devices, trackerProcessorState$.value, controlledAudioDevices, e2eeLivekitOptions) + ); + room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => { + logger.error("Failed to set E2EE enabled on room", e); + }); + + super(room, args); + + // Setup track processor syncing (blur) + this.observeTrackProcessors(scope, room, trackerProcessorState$); + // Observe mute state changes and update LiveKit microphone/camera states accordingly + this.observeMuteStates(scope); + // Observe media device changes and update LiveKit active devices accordingly + this.observeMediaDevices(scope, devices, controlledAudioDevices); + + this.workaroundRestartAudioInputTrackChrome(devices, scope); + } /** * Start the connection to LiveKit and publish local tracks. @@ -56,86 +102,53 @@ export class PublishConnection extends Connection { } }; + /// Private methods - /** - * Creates a new PublishConnection. - * @param args - The connection options. {@link ConnectionOpts} - * @param devices - The media devices to use for audio and video input. - * @param muteStates - The mute states for audio and video. - * @param e2eeLivekitOptions - The E2EE options to use for the LiveKit room. Use to share the same key provider across connections!. - * @param trackerProcessorState$ - The processor state for the video track processor (e.g. background blur). - */ - public constructor( - args: ConnectionOpts, - devices: MediaDevices, - private readonly muteStates: MuteStates, - e2eeLivekitOptions: E2EEOptions | undefined, - trackerProcessorState$: Behavior - ) { - const { scope } = args; - logger.info("[LivekitRoom] Create LiveKit room"); - const { controlledAudioDevices } = getUrlParams(); - - const factory = args.livekitRoomFactory ?? ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); - const room = factory({ - ...defaultLiveKitOptions, - videoCaptureDefaults: { - ...defaultLiveKitOptions.videoCaptureDefaults, - deviceId: devices.videoInput.selected$.value?.id, - processor: trackerProcessorState$.value.processor - }, - audioCaptureDefaults: { - ...defaultLiveKitOptions.audioCaptureDefaults, - deviceId: devices.audioInput.selected$.value?.id - }, - audioOutput: { - // When using controlled audio devices, we don't want to set the - // deviceId here, because it will be set by the native app. - // (also the id does not need to match a browser device id) - deviceId: controlledAudioDevices - ? undefined - : getValue(devices.audioOutput.selected$)?.id - }, - e2ee: e2eeLivekitOptions - }); - room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => { - logger.error("Failed to set E2EE enabled on room", e); - }); - - super(room, args); + // Restart the audio input track whenever we detect that the active media + // device has changed to refer to a different hardware device. We do this + // for the sake of Chrome, which provides a "default" device that is meant + // to match the system's default audio input, whatever that may be. + // This is special-cased for only audio inputs because we need to dig around + // in the LocalParticipant object for the track object and there's not a nice + // way to do that generically. There is usually no OS-level default video capture + // device anyway, and audio outputs work differently. + private workaroundRestartAudioInputTrackChrome(devices: MediaDevices, scope: ObservableScope): void { - // Setup track processor syncing (blur) - const track$ = scope.behavior( - observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( - map((trackRef) => { - const track = trackRef?.publication?.track; - return track instanceof LocalVideoTrack ? track : null; - }) + devices.audioInput.selected$ + .pipe( + switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), + scope.bind() ) - ); - trackProcessorSync(track$, trackerProcessorState$); + .subscribe(() => { + if (this.livekitRoom.state != ConnectionState.Connected) return; + const activeMicTrack = Array.from( + this.livekitRoom.localParticipant.audioTrackPublications.values() + ).find((d) => d.source === Track.Source.Microphone)?.track; - this.muteStates.audio.setHandler(async (desired) => { - try { - await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); - } catch (e) { - logger.error("Failed to update LiveKit audio input mute state", e); - } - return this.livekitRoom.localParticipant.isMicrophoneEnabled; - }); - this.muteStates.video.setHandler(async (desired) => { - try { - await this.livekitRoom.localParticipant.setCameraEnabled(desired); - } catch (e) { - logger.error("Failed to update LiveKit video input mute state", e); - } - return this.livekitRoom.localParticipant.isCameraEnabled; - }); - scope.onEnd(() => { - this.muteStates.audio.unsetHandler(); - this.muteStates.video.unsetHandler(); - }); + if ( + activeMicTrack && + // only restart if the stream is still running: LiveKit will detect + // when a track stops & restart appropriately, so this is not our job. + // Plus, we need to avoid restarting again if the track is already in + // the process of being restarted. + activeMicTrack.mediaStreamTrack.readyState !== "ended" + ) { + // Restart the track, which will cause Livekit to do another + // getUserMedia() call with deviceId: default to get the *new* default device. + // Note that room.switchActiveDevice() won't work: Livekit will ignore it because + // the deviceId hasn't changed (was & still is default). + this.livekitRoom.localParticipant + .getTrackPublication(Track.Source.Microphone) + ?.audioTrack?.restartTrack() + .catch((e) => { + logger.error(`Failed to restart audio device track`, e); + }); + } + }); + } +// Observe changes in the selected media devices and update the LiveKit room accordingly. + private observeMediaDevices(scope: ObservableScope, devices: MediaDevices, controlledAudioDevices: boolean):void { const syncDevice = ( kind: MediaDeviceKind, selected$: Observable @@ -165,44 +178,77 @@ export class PublishConnection extends Connection { if (!controlledAudioDevices) syncDevice("audiooutput", devices.audioOutput.selected$); syncDevice("videoinput", devices.videoInput.selected$); - // Restart the audio input track whenever we detect that the active media - // device has changed to refer to a different hardware device. We do this - // for the sake of Chrome, which provides a "default" device that is meant - // to match the system's default audio input, whatever that may be. - // This is special-cased for only audio inputs because we need to dig around - // in the LocalParticipant object for the track object and there's not a nice - // way to do that generically. There is usually no OS-level default video capture - // device anyway, and audio outputs work differently. - devices.audioInput.selected$ - .pipe( - switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), - scope.bind() - ) - .subscribe(() => { - if (this.livekitRoom.state != ConnectionState.Connected) return; - const activeMicTrack = Array.from( - this.livekitRoom.localParticipant.audioTrackPublications.values() - ).find((d) => d.source === Track.Source.Microphone)?.track; + } - if ( - activeMicTrack && - // only restart if the stream is still running: LiveKit will detect - // when a track stops & restart appropriately, so this is not our job. - // Plus, we need to avoid restarting again if the track is already in - // the process of being restarted. - activeMicTrack.mediaStreamTrack.readyState !== "ended" - ) { - // Restart the track, which will cause Livekit to do another - // getUserMedia() call with deviceId: default to get the *new* default device. - // Note that room.switchActiveDevice() won't work: Livekit will ignore it because - // the deviceId hasn't changed (was & still is default). - this.livekitRoom.localParticipant - .getTrackPublication(Track.Source.Microphone) - ?.audioTrack?.restartTrack() - .catch((e) => { - logger.error(`Failed to restart audio device track`, e); - }); - } - }); + /** + * Observe changes in the mute states and update the LiveKit room accordingly. + * @param scope + * @private + */ + private observeMuteStates(scope: ObservableScope): void { + this.muteStates.audio.setHandler(async (desired) => { + try { + await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired); + } catch (e) { + logger.error("Failed to update LiveKit audio input mute state", e); + } + return this.livekitRoom.localParticipant.isMicrophoneEnabled; + }); + this.muteStates.video.setHandler(async (desired) => { + try { + await this.livekitRoom.localParticipant.setCameraEnabled(desired); + } catch (e) { + logger.error("Failed to update LiveKit video input mute state", e); + } + return this.livekitRoom.localParticipant.isCameraEnabled; + }); + scope.onEnd(() => { + this.muteStates.audio.unsetHandler(); + this.muteStates.video.unsetHandler(); + }); } + + private observeTrackProcessors(scope: ObservableScope, room: LivekitRoom, trackerProcessorState$: Behavior): void { + const track$ = scope.behavior( + observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( + map((trackRef) => { + const track = trackRef?.publication?.track; + return track instanceof LocalVideoTrack ? track : null; + }) + ) + ); + trackProcessorSync(track$, trackerProcessorState$); + } + +} + + +// Generate the initial LiveKit RoomOptions based on the current media devices and processor state. +function generateRoomOption( + devices: MediaDevices, + processorState: ProcessorState, + controlledAudioDevices: boolean, + e2eeLivekitOptions: E2EEOptions | undefined, +): RoomOptions { + return { + ...defaultLiveKitOptions, + videoCaptureDefaults: { + ...defaultLiveKitOptions.videoCaptureDefaults, + deviceId: devices.videoInput.selected$.value?.id, + processor: processorState.processor + }, + audioCaptureDefaults: { + ...defaultLiveKitOptions.audioCaptureDefaults, + deviceId: devices.audioInput.selected$.value?.id + }, + audioOutput: { + // When using controlled audio devices, we don't want to set the + // deviceId here, because it will be set by the native app. + // (also the id does not need to match a browser device id) + deviceId: controlledAudioDevices + ? undefined + : getValue(devices.audioOutput.selected$)?.id + }, + e2ee: e2eeLivekitOptions + }; } From 91a366fa2a92de9853bbae464c17c44a229686d3 Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 6 Oct 2025 10:50:10 +0200 Subject: [PATCH 13/46] tests: Publish connection states --- src/state/Connection.test.ts | 184 +++++++++++++++++++++++++++++++---- 1 file changed, 164 insertions(+), 20 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 5529e588b..5f1778b03 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -5,18 +5,30 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { afterEach, describe, expect, it, type MockedObject, vi } from "vitest"; +import { afterEach, describe, expect, it, type Mock, Mocked, type MockedObject, vi } from "vitest"; import { type CallMembership, type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc"; -import { BehaviorSubject } from "rxjs"; -import { ConnectionState, type RemoteParticipant, type Room as LivekitRoom, RoomEvent } from "livekit-client"; +import { BehaviorSubject, of } from "rxjs"; +import { + ConnectionState, + type LocalParticipant, + type RemoteParticipant, + type Room as LivekitRoom, + RoomEvent, type RoomOptions +} from "livekit-client"; import fetchMock from "fetch-mock"; import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; +import { type BackgroundOptions, type ProcessorWrapper } from "@livekit/track-processors"; import { type ConnectionOpts, type FocusConnectionState, RemoteConnection } from "./Connection.ts"; import { ObservableScope } from "./ObservableScope.ts"; import { type OpenIDClientParts } from "../livekit/openIDSFU.ts"; import { FailToGetOpenIdToken } from "../utils/errors.ts"; +import { PublishConnection } from "./PublishConnection.ts"; +import { mockMediaDevices, mockMuteStates } from "../utils/test.ts"; +import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx"; +import { type MuteStates } from "./MuteStates.ts"; +import { DeviceLabel, MediaDevice, SelectedDevice } from "./MediaDevices.ts"; let testScope: ObservableScope; @@ -25,6 +37,9 @@ let client: MockedObject; let fakeLivekitRoom: MockedObject; +let localParticipantEventEmiter: EventEmitter; +let fakeLocalParticipant: MockedObject; + let fakeRoomEventEmiter: EventEmitter; let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>; @@ -49,18 +64,32 @@ function setupTest(): void { } as unknown as OpenIDClientParts); fakeMembershipsFocusMap$ = new BehaviorSubject<{ membership: CallMembership; focus: LivekitFocus }[]>([]); + localParticipantEventEmiter = new EventEmitter(); + + fakeLocalParticipant = vi.mocked({ + identity: "@me:example.org", + isMicrophoneEnabled: vi.fn().mockReturnValue(true), + getTrackPublication: vi.fn().mockReturnValue(undefined), + on: localParticipantEventEmiter.on.bind(localParticipantEventEmiter), + off: localParticipantEventEmiter.off.bind(localParticipantEventEmiter), + addListener: localParticipantEventEmiter.addListener.bind(localParticipantEventEmiter), + removeListener: localParticipantEventEmiter.removeListener.bind(localParticipantEventEmiter), + removeAllListeners: localParticipantEventEmiter.removeAllListeners.bind(localParticipantEventEmiter) + } as unknown as LocalParticipant); fakeRoomEventEmiter = new EventEmitter(); fakeLivekitRoom = vi.mocked({ connect: vi.fn(), disconnect: vi.fn(), remoteParticipants: new Map(), + localParticipant: fakeLocalParticipant, state: ConnectionState.Disconnected, on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter), off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), removeListener: fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), - removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter) + removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), + setE2EEEnabled: vi.fn().mockResolvedValue(undefined) } as unknown as LivekitRoom); } @@ -424,7 +453,7 @@ function fakeRemoteLivekitParticipant(id: string): RemoteParticipant { function fakeRtcMemberShip(userId: string, deviceId: string): CallMembership { return vi.mocked({ sender: userId, - deviceId: deviceId, + deviceId: deviceId } as unknown as CallMembership); } @@ -440,19 +469,19 @@ describe("Publishing participants observations", () => { const danIsAPublisher = Promise.withResolvers(); const observedPublishers: { participant: RemoteParticipant; membership: CallMembership }[][] = []; connection.publishingParticipants$.subscribe((publishers) => { - observedPublishers.push(publishers); - if (publishers.some((p) => p.participant.identity === "@bob:example.org:DEV111")) { - bobIsAPublisher.resolve(); - } - if (publishers.some((p) => p.participant.identity === "@dan:example.org:DEV333")) { - danIsAPublisher.resolve(); - } + observedPublishers.push(publishers); + if (publishers.some((p) => p.participant.identity === "@bob:example.org:DEV111")) { + bobIsAPublisher.resolve(); + } + if (publishers.some((p) => p.participant.identity === "@dan:example.org:DEV333")) { + danIsAPublisher.resolve(); + } }); // The publishingParticipants$ observable is derived from the current members of the // livekitRoom and the rtc membership in order to publish the members that are publishing // on this connection. - let participants: RemoteParticipant[]= [ + let participants: RemoteParticipant[] = [ fakeRemoteLivekitParticipant("@alice:example.org:DEV000"), fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), fakeRemoteLivekitParticipant("@carol:example.org:DEV222"), @@ -477,7 +506,7 @@ describe("Publishing participants observations", () => { livekit_alias: "!roomID:example.org", livekit_service_url: "https://other-matrix-rtc.example.org/livekit/jwt", type: "livekit" - } + }; const rtcMemberships = [ @@ -485,7 +514,7 @@ describe("Publishing participants observations", () => { { membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), focus: livekitFocus }, // Alice and carol is on a different focus { membership: fakeRtcMemberShip("@alice:example.org", "DEV000"), focus: otherFocus }, - { membership: fakeRtcMemberShip("@carol:example.org", "DEV222"), focus: otherFocus }, + { membership: fakeRtcMemberShip("@carol:example.org", "DEV222"), focus: otherFocus } // NO DAVE YET ]; // signal this change in rtc memberships @@ -520,7 +549,7 @@ describe("Publishing participants observations", () => { const updatedPublishers = observedPublishers.pop(); expect(updatedPublishers?.length).toEqual(1); expect(updatedPublishers?.some((p) => p.participant.identity === "@dan:example.org:DEV333")).toBeTruthy(); - }) + }); it("should be scoped to parent scope", async () => { @@ -533,8 +562,8 @@ describe("Publishing participants observations", () => { observedPublishers.push(publishers); }); - let participants: RemoteParticipant[]= [ - fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + let participants: RemoteParticipant[] = [ + fakeRemoteLivekitParticipant("@bob:example.org:DEV111") ]; // Let's simulate 3 members on the livekitRoom @@ -552,7 +581,7 @@ describe("Publishing participants observations", () => { const rtcMemberships = [ // Say bob is on the same focus - { membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), focus: livekitFocus }, + { membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), focus: livekitFocus } ]; // signal this change in rtc memberships fakeMembershipsFocusMap$.next(rtcMemberships); @@ -575,7 +604,122 @@ describe("Publishing participants observations", () => { fakeRoomEventEmiter.emit(RoomEvent.ParticipantDisconnected, fakeRemoteLivekitParticipant("@bob:example.org:DEV111")); expect(observedPublishers.length).toEqual(0); - }) + }); +}); + + +describe("PublishConnection", () => { + + let fakeBlurProcessor: ProcessorWrapper; + let roomFactoryMock: Mock<() => LivekitRoom>; + let muteStates: MockedObject; + + function setUpPublishConnection() { + setupTest(); + + roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom); + + + muteStates = mockMuteStates(); + + fakeBlurProcessor = vi.mocked>({ + name: "BackgroundBlur", + start: vi.fn().mockResolvedValue(undefined), + stop: vi.fn().mockResolvedValue(undefined), + setOptions: vi.fn().mockResolvedValue(undefined), + getOptions: vi.fn().mockReturnValue({ strength: 0.5 }), + isRunning: vi.fn().mockReturnValue(false) + }); + + + } + + + describe("Livekit room creation", () => { + + + function createSetup() { + setUpPublishConnection(); + const fakeTrackProcessorSubject$ = new BehaviorSubject({ + supported: true, + processor: undefined + }); + + const opts: ConnectionOpts = { + client: client, + focus: livekitFocus, + membershipsFocusMap$: fakeMembershipsFocusMap$, + scope: testScope, + livekitRoomFactory: roomFactoryMock + }; + + const audioInput = { + available$: of(new Map([["mic1", { id: "mic1" }]])), + selected$: new BehaviorSubject({ id: "mic1" }), + select(): void { + } + }; + const videoInput = { + available$: of(new Map([["cam1", { id: "cam1" }]])), + selected$: new BehaviorSubject({ id: "cam1" }), + select(): void { + } + }; + + const audioOutput = { + available$: of(new Map([["speaker", { id: "speaker" }]])), + selected$: new BehaviorSubject({ id: "speaker" }), + select(): void { + } + }; + + const fakeDevices = mockMediaDevices({ + audioInput, + videoInput, + audioOutput + }); + + new PublishConnection( + opts, + fakeDevices, + muteStates, + undefined, + fakeTrackProcessorSubject$ + ); + + } + + it("should create room with proper initial audio and video settings", () => { + + createSetup(); + + expect(roomFactoryMock).toHaveBeenCalled(); + + const lastCallArgs = roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1]; + + const roomOptions = lastCallArgs.pop() as unknown as RoomOptions; + expect(roomOptions).toBeDefined(); + + expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1"); + expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1"); + expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker"); + + }); + + it("respect controlledAudioDevices", () => { + // TODO: Refactor the code to make it testable. + // The UrlParams module is a singleton has a cache and is very hard to test. + // This breaks other tests as well if not handled properly. + // vi.mock(import("./../UrlParams"), () => { + // return { + // getUrlParams: vi.fn().mockReturnValue({ + // controlledAudioDevices: true + // }) + // }; + // }); + + }); + }); }); From c3c0516f0d663b18ec3e84e3ee7c604743ba58de Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 7 Oct 2025 16:00:59 +0200 Subject: [PATCH 14/46] Lint: fix all the lint errors --- src/livekit/MatrixAudioRenderer.test.tsx | 89 ++++++++++++++++--- src/livekit/MatrixAudioRenderer.tsx | 6 +- src/main.tsx | 1 - src/room/CallEventAudioRenderer.test.tsx | 3 +- src/room/GroupCallView.test.tsx | 7 +- src/room/InCallView.test.tsx | 9 +- src/room/InCallView.tsx | 6 +- src/room/VideoPreview.test.tsx | 7 +- src/rtcSessionHelpers.test.ts | 104 ++++++++++++++--------- src/state/Async.ts | 14 +-- src/state/CallViewModel.test.ts | 7 +- src/state/CallViewModel.ts | 9 +- src/state/Connection.test.ts | 12 +-- src/state/MuteStates.ts | 11 ++- src/tile/MediaView.test.tsx | 7 +- src/utils/test.ts | 11 ++- 16 files changed, 206 insertions(+), 97 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index 4fe7d3332..e2464eb80 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details. import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { render } from "@testing-library/react"; -import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { getTrackReferenceId, type TrackReference, @@ -15,11 +14,19 @@ import { import { type RemoteAudioTrack } from "livekit-client"; import { type ReactNode } from "react"; import { useTracks } from "@livekit/components-react"; +import { of } from "rxjs"; import { testAudioContext } from "../useAudioContext.test"; import * as MediaDevicesContext from "../MediaDevicesContext"; import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer"; -import { mockMediaDevices, mockTrack } from "../utils/test"; +import { + mockLivekitRoom, + mockMatrixRoomMember, + mockMediaDevices, + mockRtcMembership, + mockTrack +} from "../utils/test"; + export const TestAudioContextConstructor = vi.fn(() => testAudioContext); @@ -52,10 +59,26 @@ const tracks = [mockTrack("test:123")]; vi.mocked(useTracks).mockReturnValue(tracks); it("should render for member", () => { + // TODO this is duplicated test setup in all tests + const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); + const carol = mockMatrixRoomMember(localRtcMember); + const p = { + id: "test:123", + participant: undefined, + member: carol + } + const livekitRoom = mockLivekitRoom( + {}, + { + remoteParticipants$: of([]), + }, + ); const { container, queryAllByTestId } = render( , ); @@ -64,12 +87,29 @@ it("should render for member", () => { }); it("should not render without member", () => { - const memberships = [ - { sender: "othermember", deviceId: "123" }, - ] as CallMembership[]; + // const memberships = [ + // { sender: "othermember", deviceId: "123" }, + // ] as CallMembership[]; + const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); + const carol = mockMatrixRoomMember(localRtcMember); + const p = { + id: "test:123", + participant: undefined, + member: carol + } + const livekitRoom = mockLivekitRoom( + {}, + { + remoteParticipants$: of([]), + }, + ); const { container, queryAllByTestId } = render( - + , ); expect(container).toBeTruthy(); @@ -77,10 +117,25 @@ it("should not render without member", () => { }); it("should not setup audioContext gain and pan if there is no need to.", () => { + const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); + const carol = mockMatrixRoomMember(localRtcMember); + const p = { + id: "test:123", + participant: undefined, + member: carol + } + const livekitRoom = mockLivekitRoom( + {}, + { + remoteParticipants$: of([]), + }, + ); render( , ); @@ -100,11 +155,25 @@ it("should setup audioContext gain and pan", () => { pan: 1, volume: 0.1, }); + const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); + const carol = mockMatrixRoomMember(localRtcMember); + const p = { + id: "test:123", + participant: undefined, + member: carol + } + const livekitRoom = mockLivekitRoom( + {}, + { + remoteParticipants$: of([]), + }, + ); render( + participants={[p]} + url={""} + livekitRoom={livekitRoom} /> , ); diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index f402b32d7..76c206c7e 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -33,7 +33,9 @@ export interface MatrixAudioRendererProps { * that are not expected to be in the rtc session. */ participants: { - participant: Participant; + id: string; + // TODO it appears to be optional as per InCallView? but what does that mean here? a rtc member not yet joined in livekit? + participant: Participant | undefined; member: RoomMember; }[]; /** @@ -82,7 +84,7 @@ export function LivekitRoomAudioRenderer({ if (loggedInvalidIdentities.current.has(identity)) return; logger.warn( `[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`, - `current members: ${participants.map((p) => p.participant.identity)}`, + `current members: ${participants.map((p) => p.participant?.identity)}`, `track will not get rendered`, ); loggedInvalidIdentities.current.add(identity); diff --git a/src/main.tsx b/src/main.tsx index e795a13cf..f27b55a41 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -11,7 +11,6 @@ Please see LICENSE in the repository root for full details. // dependency references. import "matrix-js-sdk/lib/browser-index"; -import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; import "./index.css"; import { logger } from "matrix-js-sdk/lib/logger"; diff --git a/src/room/CallEventAudioRenderer.test.tsx b/src/room/CallEventAudioRenderer.test.tsx index 40b79da45..e7d7e85a5 100644 --- a/src/room/CallEventAudioRenderer.test.tsx +++ b/src/room/CallEventAudioRenderer.test.tsx @@ -155,7 +155,8 @@ test("plays one sound when a hand is raised", () => { act(() => { handRaisedSubject$.next({ - [bobRtcMember.callId]: { + // TODO: What is this string supposed to be? + [`${bobRtcMember.sender}:${bobRtcMember.deviceId}`]: { time: new Date(), membershipEventId: "", reactionEventId: "", diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx index b8bc2f534..22d99b31c 100644 --- a/src/room/GroupCallView.test.tsx +++ b/src/room/GroupCallView.test.tsx @@ -26,7 +26,6 @@ import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-cont import { useState } from "react"; import { TooltipProvider } from "@vector-im/compound-web"; -import { type MuteStates } from "./MuteStates"; import { prefetchSounds } from "../soundUtils"; import { useAudioContext } from "../useAudioContext"; import { ActiveCall } from "./InCallView"; @@ -47,6 +46,7 @@ import { ProcessorProvider } from "../livekit/TrackProcessorContext"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { HeaderStyle } from "../UrlParams"; import { constant } from "../state/Behavior"; +import { type MuteStates } from "../state/MuteStates.ts"; vi.mock("../soundUtils"); vi.mock("../useAudioContext"); @@ -150,7 +150,7 @@ function createGroupCallView( const muteState = { audio: { enabled: false }, video: { enabled: false }, - } as MuteStates; + } as unknown as MuteStates; const { getByText } = render( @@ -164,9 +164,10 @@ function createGroupCallView( skipLobby={false} header={HeaderStyle.Standard} rtcSession={rtcSession as unknown as MatrixRTCSession} - isJoined={joined} muteStates={muteState} widget={widget} + joined={true} + setJoined={function(value: boolean): void { }} /> diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index 6d2aaf0ab..d26941202 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -24,7 +24,6 @@ import { TooltipProvider } from "@vector-im/compound-web"; import { RoomContext, useLocalParticipant } from "@livekit/components-react"; import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport"; -import { type MuteStates } from "./MuteStates"; import { InCallView } from "./InCallView"; import { mockLivekitRoom, @@ -48,6 +47,7 @@ import { useRoomEncryptionSystem } from "../e2ee/sharedKeyManagement"; import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer"; import { MediaDevicesContext } from "../MediaDevicesContext"; import { HeaderStyle } from "../UrlParams"; +import { type MuteStates } from "../state/MuteStates.ts"; // vi.hoisted(() => { // localStorage = {} as unknown as Storage; @@ -136,7 +136,7 @@ function createInCallView(): RenderResult & { const muteState = { audio: { enabled: false }, video: { enabled: false }, - } as MuteStates; + } as unknown as MuteStates; const livekitRoom = mockLivekitRoom( { localParticipant, @@ -176,11 +176,6 @@ function createInCallView(): RenderResult & { }, }} matrixRoom={room} - livekitRoom={livekitRoom} - participantCount={0} - onLeft={function (): void { - throw new Error("Function not implemented."); - }} onShareClick={null} /> diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 57873b400..8474c2fdc 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -23,7 +23,7 @@ import useMeasure from "react-use-measure"; import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import classNames from "classnames"; import { BehaviorSubject, map } from "rxjs"; -import { useObservable, useObservableEagerState } from "observable-hooks"; +import { useObservable } from "observable-hooks"; import { logger } from "matrix-js-sdk/lib/logger"; import { RoomAndToDeviceEvents } from "matrix-js-sdk/lib/matrixrtc/RoomAndToDeviceKeyTransport"; import { @@ -112,7 +112,6 @@ import { prefetchSounds } from "../soundUtils"; import { useAudioContext } from "../useAudioContext"; import ringtoneMp3 from "../sound/ringtone.mp3?url"; import ringtoneOgg from "../sound/ringtone.ogg?url"; -import { ConnectionLostError } from "../utils/errors.ts"; import { useTrackProcessorObservable$ } from "../livekit/TrackProcessorContext.tsx"; const maxTapDurationMs = 400; @@ -206,7 +205,8 @@ export const InCallView: FC = ({ useReactionsSender(); useWakeLock(); - const connectionState = useObservableEagerState(vm.livekitConnectionState$); + // TODO multi-sfu This is unused now?? + // const connectionState = useObservableEagerState(vm.livekitConnectionState$); // annoyingly we don't get the disconnection reason this way, // only by listening for the emitted event diff --git a/src/room/VideoPreview.test.tsx b/src/room/VideoPreview.test.tsx index 717333eec..17a05e344 100644 --- a/src/room/VideoPreview.test.tsx +++ b/src/room/VideoPreview.test.tsx @@ -5,12 +5,11 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { expect, describe, it, vi, beforeAll } from "vitest"; +import { expect, describe, it, beforeAll } from "vitest"; import { render } from "@testing-library/react"; import { type MatrixInfo, VideoPreview } from "./VideoPreview"; import { E2eeType } from "../e2ee/e2eeType"; -import { mockMuteStates } from "../utils/test"; describe("VideoPreview", () => { const matrixInfo: MatrixInfo = { @@ -42,7 +41,7 @@ describe("VideoPreview", () => { const { queryByRole } = render( } />, @@ -54,7 +53,7 @@ describe("VideoPreview", () => { const { queryByRole } = render( } />, diff --git a/src/rtcSessionHelpers.test.ts b/src/rtcSessionHelpers.test.ts index 2ef9e3f10..1058628f8 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/rtcSessionHelpers.test.ts @@ -23,37 +23,38 @@ vi.mock("./widget", () => ({ ...actualWidget, widget: { api: { - setAlwaysOnScreen: (): void => {}, - transport: { send: vi.fn(), reply: vi.fn(), stop: vi.fn() }, + setAlwaysOnScreen: (): void => { + }, + transport: { send: vi.fn(), reply: vi.fn(), stop: vi.fn() } }, - lazyActions: new EventEmitter(), - }, + lazyActions: new EventEmitter() + } })); test("It joins the correct Session", async () => { const focusFromOlderMembership = { type: "livekit", livekit_service_url: "http://my-oldest-member-service-url.com", - livekit_alias: "my-oldest-member-service-alias", + livekit_alias: "my-oldest-member-service-alias" }; const focusConfigFromWellKnown = { type: "livekit", - livekit_service_url: "http://my-well-known-service-url.com", + livekit_service_url: "http://my-well-known-service-url.com" }; const focusConfigFromWellKnown2 = { type: "livekit", - livekit_service_url: "http://my-well-known-service-url2.com", + livekit_service_url: "http://my-well-known-service-url2.com" }; const clientWellKnown = { "org.matrix.msc4143.rtc_foci": [ focusConfigFromWellKnown, - focusConfigFromWellKnown2, - ], + focusConfigFromWellKnown2 + ] }; mockConfig({ - livekit: { livekit_service_url: "http://my-default-service-url.com" }, + livekit: { livekit_service_url: "http://my-default-service-url.com" } }); vi.spyOn(AutoDiscovery, "getRawClientConfig").mockImplementation( @@ -62,7 +63,7 @@ test("It joins the correct Session", async () => { return Promise.resolve(clientWellKnown); } return Promise.resolve({}); - }, + } ); const mockedSession = vi.mocked({ @@ -74,58 +75,64 @@ test("It joins the correct Session", async () => { access_token: "ACCCESS_TOKEN", token_type: "Bearer", matrix_server_name: "localhost", - expires_in: 10000, - }), - }, + expires_in: 10000 + }) + } }, memberships: [], getFocusInUse: vi.fn().mockReturnValue(focusFromOlderMembership), getOldestMembership: vi.fn().mockReturnValue({ - getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]), + getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]) }), - joinRoomSession: vi.fn(), + joinRoomSession: vi.fn() }) as unknown as MatrixRTCSession; - await enterRTCSession(mockedSession, false); + + await enterRTCSession(mockedSession, { + livekit_alias: "roomId", + livekit_service_url: "http://my-well-known-service-url.com", + type: "livekit" + }, + true); expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith( [ { livekit_alias: "my-oldest-member-service-alias", livekit_service_url: "http://my-oldest-member-service-url.com", - type: "livekit", + type: "livekit" }, { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url.com", - type: "livekit", + type: "livekit" }, { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url2.com", - type: "livekit", + type: "livekit" }, { livekit_alias: "roomId", livekit_service_url: "http://my-default-service-url.com", - type: "livekit", - }, + type: "livekit" + } ], { focus_selection: "oldest_membership", - type: "livekit", + type: "livekit" }, { manageMediaKeys: false, useLegacyMemberEvents: false, useNewMembershipManager: true, - useExperimentalToDeviceTransport: false, - }, + useExperimentalToDeviceTransport: false + } ); }); async function testLeaveRTCSession( cause: "user" | "error", - expectClose: boolean, + expectClose: boolean ): Promise { vi.clearAllMocks(); const session = { leaveRoomSession: vi.fn() } as unknown as MatrixRTCSession; @@ -133,18 +140,18 @@ async function testLeaveRTCSession( expect(session.leaveRoomSession).toHaveBeenCalled(); expect(widget!.api.transport.send).toHaveBeenCalledWith( ElementWidgetActions.HangupCall, - expect.anything(), + expect.anything() ); if (expectClose) { expect(widget!.api.transport.send).toHaveBeenCalledWith( ElementWidgetActions.Close, - expect.anything(), + expect.anything() ); expect(widget!.api.transport.stop).toHaveBeenCalled(); } else { expect(widget!.api.transport.send).not.toHaveBeenCalledWith( ElementWidgetActions.Close, - expect.anything(), + expect.anything() ); expect(widget!.api.transport.stop).not.toHaveBeenCalled(); } @@ -172,16 +179,24 @@ test("It fails with configuration error if no live kit url config is set in fall room: { roomId: "roomId", client: { - getDomain: vi.fn().mockReturnValue("example.org"), - }, + getDomain: vi.fn().mockReturnValue("example.org") + } }, memberships: [], getFocusInUse: vi.fn(), - joinRoomSession: vi.fn(), + joinRoomSession: vi.fn() }) as unknown as MatrixRTCSession; - await expect(enterRTCSession(mockedSession, false)).rejects.toThrowError( - expect.objectContaining({ code: ErrorCode.MISSING_MATRIX_RTC_FOCUS }), + await expect(enterRTCSession( + mockedSession, + { + livekit_alias: "roomId", + livekit_service_url: "http://my-well-known-service-url.com", + type: "livekit" + }, + true + )).rejects.toThrowError( + expect.objectContaining({ code: ErrorCode.MISSING_MATRIX_RTC_TRANSPORT }) ); }); @@ -191,9 +206,9 @@ test("It should not fail with configuration error if homeserver config has livek "org.matrix.msc4143.rtc_foci": [ { type: "livekit", - livekit_service_url: "http://my-well-known-service-url.com", - }, - ], + livekit_service_url: "http://my-well-known-service-url.com" + } + ] }); const mockedSession = vi.mocked({ @@ -205,14 +220,19 @@ test("It should not fail with configuration error if homeserver config has livek access_token: "ACCCESS_TOKEN", token_type: "Bearer", matrix_server_name: "localhost", - expires_in: 10000, - }), - }, + expires_in: 10000 + }) + } }, memberships: [], getFocusInUse: vi.fn(), - joinRoomSession: vi.fn(), + joinRoomSession: vi.fn() }) as unknown as MatrixRTCSession; - await enterRTCSession(mockedSession, false); + await enterRTCSession(mockedSession, { + livekit_alias: "roomId", + livekit_service_url: "http://my-well-known-service-url.com", + type: "livekit" + }, + true); }); diff --git a/src/state/Async.ts b/src/state/Async.ts index 2baa674c1..456767593 100644 --- a/src/state/Async.ts +++ b/src/state/Async.ts @@ -9,12 +9,13 @@ import { catchError, from, map, - Observable, + type Observable, of, - startWith, - switchMap, + startWith } from "rxjs"; +// TODO where are all the comments? ::cry:: +// There used to be an unitialized state!, a state might not start in loading export type Async = | { state: "loading" } | { state: "error"; value: Error } @@ -24,21 +25,22 @@ export const loading: Async = { state: "loading" }; export function error(value: Error): Async { return { state: "error", value }; } + export function ready(value: A): Async { return { state: "ready", value }; } -export function async(promise: Promise): Observable> { +export function async$(promise: Promise): Observable> { return from(promise).pipe( map(ready), startWith(loading), - catchError((e) => of(error(e))), + catchError((e: unknown) => of(error(e as Error ?? new Error("Unknown error")))), ); } export function mapAsync( async: Async, - project: (value: A) => B, + project: (value: A) => B ): Async { return async.state === "ready" ? ready(project(async.value)) : async; } diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index 07c78ef66..d9cad2b77 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -68,7 +68,7 @@ import { type ECConnectionState, } from "../livekit/useECConnectionState"; import { E2eeType } from "../e2ee/e2eeType"; -import type { RaisedHandInfo } from "../reactions"; +import type { RaisedHandInfo, ReactionInfo } from "../reactions"; import { alice, aliceDoppelganger, @@ -95,6 +95,7 @@ import { ObservableScope } from "./ObservableScope"; import { MediaDevices } from "./MediaDevices"; import { getValue } from "../utils/observable"; import { type Behavior, constant } from "./Behavior"; +import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx"; const getUrlParams = vi.hoisted(() => vi.fn(() => ({}))); vi.mock("../UrlParams", () => ({ getUrlParams })); @@ -341,6 +342,7 @@ function withCallViewModel( .mockImplementation((_room, _eventType) => of()); const muteStates = mockMuteStates(); const raisedHands$ = new BehaviorSubject>({}); + const reactions$ = new BehaviorSubject>({}); const vm = new CallViewModel( rtcSession as unknown as MatrixRTCSession, @@ -349,7 +351,8 @@ function withCallViewModel( muteStates, options, raisedHands$, - new BehaviorSubject({}), + reactions$, + new BehaviorSubject({ processor: undefined, supported: undefined }), ); onTestFinished(() => { diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 2c02521e6..4b8ff879c 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -132,7 +132,7 @@ import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; import { ElementWidgetActions, widget } from "../widget"; import { PublishConnection } from "./PublishConnection.ts"; -import { type Async, async, mapAsync, ready } from "./Async"; +import { type Async, async$, mapAsync, ready } from "./Async"; export interface CallViewModelOptions { encryptionSystem: EncryptionSystem; @@ -520,7 +520,7 @@ export class CallViewModel extends ViewModel { joined ? combineLatest( [ - async(this.preferredTransport), + async$(this.preferredTransport), this.memberships$, multiSfu.value$, ], @@ -1953,7 +1953,10 @@ export class CallViewModel extends ViewModel { .subscribe(({ start, stop }) => { for (const c of stop) { logger.info(`Disconnecting from ${c.localTransport.livekit_service_url}`); - c.stop(); + c.stop().catch((err) => { + // TODO: better error handling + logger.error("MuteState: handler error", err); + });; } for (const c of start) { c.start().then( diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 07a38d7db..74a615156 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -6,7 +6,6 @@ Please see LICENSE in the repository root for full details. */ import { afterEach, describe, expect, it, type Mock, type MockedObject, vi } from "vitest"; -import type { CallMembership, LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, of } from "rxjs"; import { ConnectionState, @@ -18,8 +17,8 @@ import { import fetchMock from "fetch-mock"; import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; -import { type BackgroundOptions, type ProcessorWrapper } from "@livekit/track-processors"; +import type { CallMembership, LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; import { type ConnectionOpts, type FocusConnectionState, RemoteConnection } from "./Connection.ts"; import { ObservableScope } from "./ObservableScope.ts"; import { type OpenIDClientParts } from "../livekit/openIDSFU.ts"; @@ -29,7 +28,6 @@ import { mockMediaDevices, mockMuteStates } from "../utils/test.ts"; import type { ProcessorState } from "../livekit/TrackProcessorContext.tsx"; import { type MuteStates } from "./MuteStates.ts"; - let testScope: ObservableScope; let client: MockedObject; @@ -551,7 +549,7 @@ describe("Publishing participants observations", () => { }); - it("should be scoped to parent scope", async () => { + it("should be scoped to parent scope", (): void => { setupTest(); const connection = setupRemoteConnection(); @@ -613,7 +611,7 @@ describe("PublishConnection", () => { let roomFactoryMock: Mock<() => LivekitRoom>; let muteStates: MockedObject; - function setUpPublishConnection() { + function setUpPublishConnection(): void { setupTest(); roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom); @@ -673,9 +671,13 @@ describe("PublishConnection", () => { } }; + // TODO understand what is wrong with our mocking that requires ts-expect-error const fakeDevices = mockMediaDevices({ + // @ts-expect-error Mocking only audioInput, + // @ts-expect-error Mocking only videoInput, + // @ts-expect-error Mocking only audioOutput }); diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index 07bc5665e..8a0258825 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -88,7 +88,10 @@ class MuteState { } else { subscriber.next(enabled); syncing = true; - sync(); + sync().catch((err) => { + // TODO: better error handling + logger.error("MuteState: handler error", err); + }); } } }; @@ -97,7 +100,10 @@ class MuteState { latestDesired = desired; if (syncing === false) { syncing = true; - sync(); + sync().catch((err) => { + // TODO: better error handling + logger.error("MuteState: handler error", err); + }); } }); return (): void => s.unsubscribe(); @@ -132,6 +138,7 @@ class MuteState { ) {} } +// TODO there is another MuteStates in src/room/MuteStates.tsx ?? why export class MuteStates { public readonly audio = new MuteState( this.scope, diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index 672f33344..57be00ef5 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -19,7 +19,7 @@ import { type ComponentProps } from "react"; import { MediaView } from "./MediaView"; import { EncryptionStatus } from "../state/MediaViewModel"; -import { mockLocalParticipant } from "../utils/test"; +import { mockLocalParticipant, mockMatrixRoomMember, mockRtcMembership } from "../utils/test"; describe("MediaView", () => { const participant = mockLocalParticipant({}); @@ -45,7 +45,10 @@ describe("MediaView", () => { mirror: false, unencryptedWarning: false, video: trackReference, - member: undefined, + member: mockMatrixRoomMember( + mockRtcMembership("@alice:example.org", "CCCC"), + { name: "some name" }, + ), localParticipant: false, focusable: true, }; diff --git a/src/utils/test.ts b/src/utils/test.ts index 519fdd509..b77e63c00 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { map, type Observable, of, type SchedulerLike } from "rxjs"; import { type RunHelpers, TestScheduler } from "rxjs/testing"; -import { expect, vi, vitest } from "vitest"; +import { expect, type MockedObject, vi, vitest } from "vitest"; import { type RoomMember, type Room as MatrixRoom, @@ -205,6 +205,9 @@ export function mockMatrixRoomMember( return { ...mockEmitter(), userId: rtcMembership.sender, + getMxcAvatarUrl(): string | undefined { + return undefined; + }, ...member, } as RoomMember; } @@ -416,13 +419,13 @@ export const deviceStub = { select(): void {}, }; -export function mockMediaDevices(data: Partial): MediaDevices { - return { +export function mockMediaDevices(data: Partial): MockedObject { + return vi.mocked({ audioInput: deviceStub, audioOutput: deviceStub, videoInput: deviceStub, ...data, - } as MediaDevices; + } as MediaDevices); } export function mockMuteStates( From c820ba39837d04b9c16d41b975bd60bb77bfa310 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 7 Oct 2025 16:07:46 +0200 Subject: [PATCH 15/46] build: update lock file --- yarn.lock | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/yarn.lock b/yarn.lock index 4429b7d49..912a13a2e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5176,6 +5176,13 @@ __metadata: languageName: node linkType: hard +"@types/glob-to-regexp@npm:^0.4.4": + version: 0.4.4 + resolution: "@types/glob-to-regexp@npm:0.4.4" + checksum: 10c0/7288ff853850d8302a8770a3698b187fc3970ad12ee6427f0b3758a3e7a0ebb0bd993abc6ebaaa979d09695b4194157d2bfaa7601b0fb9ed72c688b4c1298b88 + languageName: node + linkType: hard + "@types/grecaptcha@npm:^3.0.9": version: 3.0.9 resolution: "@types/grecaptcha@npm:3.0.9" @@ -7528,6 +7535,7 @@ __metadata: eslint-plugin-react-hooks: "npm:^5.0.0" eslint-plugin-rxjs: "npm:^5.0.3" eslint-plugin-unicorn: "npm:^56.0.0" + fetch-mock: "npm:11.1.5" global-jsdom: "npm:^26.0.0" i18next: "npm:^24.0.0" i18next-browser-languagedetector: "npm:^8.0.0" @@ -8495,6 +8503,22 @@ __metadata: languageName: node linkType: hard +"fetch-mock@npm:11.1.5": + version: 11.1.5 + resolution: "fetch-mock@npm:11.1.5" + dependencies: + "@types/glob-to-regexp": "npm:^0.4.4" + dequal: "npm:^2.0.3" + glob-to-regexp: "npm:^0.4.1" + is-subset: "npm:^0.1.1" + regexparam: "npm:^3.0.0" + peerDependenciesMeta: + node-fetch: + optional: true + checksum: 10c0/f32f1d7879b654a3fab7c3576901193ddd4c63cb9aeae2ed66ff42062400c0937d4696b1a5171e739d5f62470e6554e190f14816789f5e3b2bf1ad90208222e6 + languageName: node + linkType: hard + "fflate@npm:^0.4.8": version: 0.4.8 resolution: "fflate@npm:0.4.8" @@ -8876,6 +8900,13 @@ __metadata: languageName: node linkType: hard +"glob-to-regexp@npm:^0.4.1": + version: 0.4.1 + resolution: "glob-to-regexp@npm:0.4.1" + checksum: 10c0/0486925072d7a916f052842772b61c3e86247f0a80cc0deb9b5a3e8a1a9faad5b04fb6f58986a09f34d3e96cd2a22a24b7e9882fb1cf904c31e9a310de96c429 + languageName: node + linkType: hard + "glob@npm:^10.2.2, glob@npm:^10.3.10, glob@npm:^10.3.7, glob@npm:^10.4.1": version: 10.4.5 resolution: "glob@npm:10.4.5" @@ -9611,6 +9642,13 @@ __metadata: languageName: node linkType: hard +"is-subset@npm:^0.1.1": + version: 0.1.1 + resolution: "is-subset@npm:0.1.1" + checksum: 10c0/d8125598ab9077a76684e18726fb915f5cea7a7358ed0c6ff723f4484d71a0a9981ee5aae06c44de99cfdef0fefce37438c6257ab129e53c82045ea0c2acdebf + languageName: node + linkType: hard + "is-symbol@npm:^1.0.4, is-symbol@npm:^1.1.1": version: 1.1.1 resolution: "is-symbol@npm:1.1.1" @@ -10299,7 +10337,7 @@ __metadata: "matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=voip-team/multi-SFU": version: 38.3.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=ca4a9c655537702daf9a69ed5d94831cebc49666" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=4643844597f8bd0196714ecc1c7fafd3f3f6669d" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0" @@ -10315,7 +10353,7 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/1fb0933d0bb686b0f290b1a62f75eec290b7c52a410d5968c2ccfb527a64e78a58012e1bd8f90c874d385dace3228b9a8c80e114ee227fc8a60e7c9611112ceb + checksum: 10c0/90d6feb7c5214b2fce7b8d6394c88d39a538224dac464e2a5315fb63388126999da28284bc9b7443e035ad0f24c21c1c7d9e1ad4245ee854595e73a390f48c2a languageName: node linkType: hard @@ -12043,6 +12081,13 @@ __metadata: languageName: node linkType: hard +"regexparam@npm:^3.0.0": + version: 3.0.0 + resolution: "regexparam@npm:3.0.0" + checksum: 10c0/a6430d7b97d5a7d5518f37a850b6b73aab479029d02f46af4fa0e8e4a1d7aad05b7a0d2d10c86ded21a14d5f0fa4c68525f873a5fca2efeefcccd93c36627459 + languageName: node + linkType: hard + "regexpu-core@npm:^6.2.0": version: 6.2.0 resolution: "regexpu-core@npm:6.2.0" From 743796119588ea329a5d09150a113308792273a8 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 7 Oct 2025 16:12:23 +0200 Subject: [PATCH 16/46] lint: fix import order --- src/state/PublishConnection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts index 6c15fc0fe..8381c0927 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/PublishConnection.ts @@ -13,12 +13,12 @@ import { Track } from "livekit-client"; import { map, NEVER, type Observable, type Subscription, switchMap } from "rxjs"; +import { logger } from "matrix-js-sdk/lib/logger"; import type { Behavior } from "./Behavior.ts"; import type { MediaDevices, SelectedDevice } from "./MediaDevices.ts"; import type { MuteStates } from "./MuteStates.ts"; import { type ProcessorState, trackProcessorSync } from "../livekit/TrackProcessorContext.tsx"; -import { logger } from "../../../matrix-js-sdk/lib/logger"; import { getUrlParams } from "../UrlParams.ts"; import { defaultLiveKitOptions } from "../livekit/options.ts"; import { getValue } from "../utils/observable.ts"; From 529cb8a7ec68908f14cbbc5fa509ae0901df4400 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 7 Oct 2025 16:24:02 +0200 Subject: [PATCH 17/46] prettier ! --- src/livekit/MatrixAudioRenderer.test.tsx | 22 +- src/room/GroupCallView.test.tsx | 2 +- src/room/RoomPage.tsx | 84 +++-- src/rtcSessionHelpers.test.ts | 117 +++--- src/state/Async.ts | 15 +- src/state/CallViewModel.test.ts | 5 +- src/state/CallViewModel.ts | 60 +-- src/state/Connection.test.ts | 443 ++++++++++++----------- src/state/Connection.ts | 131 ++++--- src/state/PublishConnection.ts | 79 ++-- src/tile/MediaView.test.tsx | 6 +- src/utils/test.ts | 4 +- 12 files changed, 547 insertions(+), 421 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index e2464eb80..075927327 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -24,10 +24,9 @@ import { mockMatrixRoomMember, mockMediaDevices, mockRtcMembership, - mockTrack + mockTrack, } from "../utils/test"; - export const TestAudioContextConstructor = vi.fn(() => testAudioContext); const MediaDevicesProvider = MediaDevicesContext.MediaDevicesContext.Provider; @@ -65,8 +64,8 @@ it("should render for member", () => { const p = { id: "test:123", participant: undefined, - member: carol - } + member: carol, + }; const livekitRoom = mockLivekitRoom( {}, { @@ -95,8 +94,8 @@ it("should not render without member", () => { const p = { id: "test:123", participant: undefined, - member: carol - } + member: carol, + }; const livekitRoom = mockLivekitRoom( {}, { @@ -122,8 +121,8 @@ it("should not setup audioContext gain and pan if there is no need to.", () => { const p = { id: "test:123", participant: undefined, - member: carol - } + member: carol, + }; const livekitRoom = mockLivekitRoom( {}, { @@ -160,8 +159,8 @@ it("should setup audioContext gain and pan", () => { const p = { id: "test:123", participant: undefined, - member: carol - } + member: carol, + }; const livekitRoom = mockLivekitRoom( {}, { @@ -173,7 +172,8 @@ it("should setup audioContext gain and pan", () => { + livekitRoom={livekitRoom} + /> , ); diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx index 22d99b31c..8c4a276ae 100644 --- a/src/room/GroupCallView.test.tsx +++ b/src/room/GroupCallView.test.tsx @@ -167,7 +167,7 @@ function createGroupCallView( muteStates={muteState} widget={widget} joined={true} - setJoined={function(value: boolean): void { }} + setJoined={function (value: boolean): void {}} /> diff --git a/src/room/RoomPage.tsx b/src/room/RoomPage.tsx index 3924437bb..e9527e032 100644 --- a/src/room/RoomPage.tsx +++ b/src/room/RoomPage.tsx @@ -116,20 +116,22 @@ export const RoomPage: FC = () => { const groupCallView = (): ReactNode => { switch (groupCallState.kind) { case "loaded": - return muteStates && ( - + return ( + muteStates && ( + + ) ); case "waitForInvite": case "canKnock": { @@ -148,31 +150,35 @@ export const RoomPage: FC = () => { ); return ( - muteStates && knock?.()} - enterLabel={label} - waitingForInvite={groupCallState.kind === "waitForInvite"} - confineToRoom={confineToRoom} - hideHeader={header !== "standard"} - participantCount={null} - muteStates={muteStates} - onShareClick={null} - /> + muteStates && ( + knock?.()} + enterLabel={label} + waitingForInvite={groupCallState.kind === "waitForInvite"} + confineToRoom={confineToRoom} + hideHeader={header !== "standard"} + participantCount={null} + muteStates={muteStates} + onShareClick={null} + /> + ) ); } case "loading": diff --git a/src/rtcSessionHelpers.test.ts b/src/rtcSessionHelpers.test.ts index 1058628f8..258d2f9a2 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/rtcSessionHelpers.test.ts @@ -23,38 +23,37 @@ vi.mock("./widget", () => ({ ...actualWidget, widget: { api: { - setAlwaysOnScreen: (): void => { - }, - transport: { send: vi.fn(), reply: vi.fn(), stop: vi.fn() } + setAlwaysOnScreen: (): void => {}, + transport: { send: vi.fn(), reply: vi.fn(), stop: vi.fn() }, }, - lazyActions: new EventEmitter() - } + lazyActions: new EventEmitter(), + }, })); test("It joins the correct Session", async () => { const focusFromOlderMembership = { type: "livekit", livekit_service_url: "http://my-oldest-member-service-url.com", - livekit_alias: "my-oldest-member-service-alias" + livekit_alias: "my-oldest-member-service-alias", }; const focusConfigFromWellKnown = { type: "livekit", - livekit_service_url: "http://my-well-known-service-url.com" + livekit_service_url: "http://my-well-known-service-url.com", }; const focusConfigFromWellKnown2 = { type: "livekit", - livekit_service_url: "http://my-well-known-service-url2.com" + livekit_service_url: "http://my-well-known-service-url2.com", }; const clientWellKnown = { "org.matrix.msc4143.rtc_foci": [ focusConfigFromWellKnown, - focusConfigFromWellKnown2 - ] + focusConfigFromWellKnown2, + ], }; mockConfig({ - livekit: { livekit_service_url: "http://my-default-service-url.com" } + livekit: { livekit_service_url: "http://my-default-service-url.com" }, }); vi.spyOn(AutoDiscovery, "getRawClientConfig").mockImplementation( @@ -63,7 +62,7 @@ test("It joins the correct Session", async () => { return Promise.resolve(clientWellKnown); } return Promise.resolve({}); - } + }, ); const mockedSession = vi.mocked({ @@ -75,64 +74,67 @@ test("It joins the correct Session", async () => { access_token: "ACCCESS_TOKEN", token_type: "Bearer", matrix_server_name: "localhost", - expires_in: 10000 - }) - } + expires_in: 10000, + }), + }, }, memberships: [], getFocusInUse: vi.fn().mockReturnValue(focusFromOlderMembership), getOldestMembership: vi.fn().mockReturnValue({ - getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]) + getPreferredFoci: vi.fn().mockReturnValue([focusFromOlderMembership]), }), - joinRoomSession: vi.fn() + joinRoomSession: vi.fn(), }) as unknown as MatrixRTCSession; - await enterRTCSession(mockedSession, { + await enterRTCSession( + mockedSession, + { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url.com", - type: "livekit" + type: "livekit", }, - true); + true, + ); expect(mockedSession.joinRoomSession).toHaveBeenLastCalledWith( [ { livekit_alias: "my-oldest-member-service-alias", livekit_service_url: "http://my-oldest-member-service-url.com", - type: "livekit" + type: "livekit", }, { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url.com", - type: "livekit" + type: "livekit", }, { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url2.com", - type: "livekit" + type: "livekit", }, { livekit_alias: "roomId", livekit_service_url: "http://my-default-service-url.com", - type: "livekit" - } + type: "livekit", + }, ], { focus_selection: "oldest_membership", - type: "livekit" + type: "livekit", }, { manageMediaKeys: false, useLegacyMemberEvents: false, useNewMembershipManager: true, - useExperimentalToDeviceTransport: false - } + useExperimentalToDeviceTransport: false, + }, ); }); async function testLeaveRTCSession( cause: "user" | "error", - expectClose: boolean + expectClose: boolean, ): Promise { vi.clearAllMocks(); const session = { leaveRoomSession: vi.fn() } as unknown as MatrixRTCSession; @@ -140,18 +142,18 @@ async function testLeaveRTCSession( expect(session.leaveRoomSession).toHaveBeenCalled(); expect(widget!.api.transport.send).toHaveBeenCalledWith( ElementWidgetActions.HangupCall, - expect.anything() + expect.anything(), ); if (expectClose) { expect(widget!.api.transport.send).toHaveBeenCalledWith( ElementWidgetActions.Close, - expect.anything() + expect.anything(), ); expect(widget!.api.transport.stop).toHaveBeenCalled(); } else { expect(widget!.api.transport.send).not.toHaveBeenCalledWith( ElementWidgetActions.Close, - expect.anything() + expect.anything(), ); expect(widget!.api.transport.stop).not.toHaveBeenCalled(); } @@ -179,24 +181,26 @@ test("It fails with configuration error if no live kit url config is set in fall room: { roomId: "roomId", client: { - getDomain: vi.fn().mockReturnValue("example.org") - } + getDomain: vi.fn().mockReturnValue("example.org"), + }, }, memberships: [], getFocusInUse: vi.fn(), - joinRoomSession: vi.fn() + joinRoomSession: vi.fn(), }) as unknown as MatrixRTCSession; - await expect(enterRTCSession( - mockedSession, - { - livekit_alias: "roomId", - livekit_service_url: "http://my-well-known-service-url.com", - type: "livekit" - }, - true - )).rejects.toThrowError( - expect.objectContaining({ code: ErrorCode.MISSING_MATRIX_RTC_TRANSPORT }) + await expect( + enterRTCSession( + mockedSession, + { + livekit_alias: "roomId", + livekit_service_url: "http://my-well-known-service-url.com", + type: "livekit", + }, + true, + ), + ).rejects.toThrowError( + expect.objectContaining({ code: ErrorCode.MISSING_MATRIX_RTC_TRANSPORT }), ); }); @@ -206,9 +210,9 @@ test("It should not fail with configuration error if homeserver config has livek "org.matrix.msc4143.rtc_foci": [ { type: "livekit", - livekit_service_url: "http://my-well-known-service-url.com" - } - ] + livekit_service_url: "http://my-well-known-service-url.com", + }, + ], }); const mockedSession = vi.mocked({ @@ -220,19 +224,22 @@ test("It should not fail with configuration error if homeserver config has livek access_token: "ACCCESS_TOKEN", token_type: "Bearer", matrix_server_name: "localhost", - expires_in: 10000 - }) - } + expires_in: 10000, + }), + }, }, memberships: [], getFocusInUse: vi.fn(), - joinRoomSession: vi.fn() + joinRoomSession: vi.fn(), }) as unknown as MatrixRTCSession; - await enterRTCSession(mockedSession, { + await enterRTCSession( + mockedSession, + { livekit_alias: "roomId", livekit_service_url: "http://my-well-known-service-url.com", - type: "livekit" + type: "livekit", }, - true); + true, + ); }); diff --git a/src/state/Async.ts b/src/state/Async.ts index 456767593..79de41409 100644 --- a/src/state/Async.ts +++ b/src/state/Async.ts @@ -5,14 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - catchError, - from, - map, - type Observable, - of, - startWith -} from "rxjs"; +import { catchError, from, map, type Observable, of, startWith } from "rxjs"; // TODO where are all the comments? ::cry:: // There used to be an unitialized state!, a state might not start in loading @@ -34,13 +27,15 @@ export function async$(promise: Promise): Observable> { return from(promise).pipe( map(ready), startWith(loading), - catchError((e: unknown) => of(error(e as Error ?? new Error("Unknown error")))), + catchError((e: unknown) => + of(error((e as Error) ?? new Error("Unknown error"))), + ), ); } export function mapAsync( async: Async, - project: (value: A) => B + project: (value: A) => B, ): Async { return async.state === "ready" ? ready(project(async.value)) : async; } diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index d9cad2b77..acc6a9912 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -352,7 +352,10 @@ function withCallViewModel( options, raisedHands$, reactions$, - new BehaviorSubject({ processor: undefined, supported: undefined }), + new BehaviorSubject({ + processor: undefined, + supported: undefined, + }), ); onTestFinished(() => { diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 4b8ff879c..f517908f8 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -126,7 +126,11 @@ import { } from "../rtcSessionHelpers"; import { E2eeType } from "../e2ee/e2eeType"; import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider"; -import { type Connection, type ConnectionOpts, RemoteConnection } from "./Connection"; +import { + type Connection, + type ConnectionOpts, + RemoteConnection, +} from "./Connection"; import { type MuteStates } from "./MuteStates"; import { getUrlParams } from "../UrlParams"; import { type ProcessorState } from "../livekit/TrackProcessorContext"; @@ -485,7 +489,6 @@ export class CallViewModel extends ViewModel { ), ); - /** * The MatrixRTC session participants. */ @@ -574,7 +577,6 @@ export class CallViewModel extends ViewModel { (transport) => transport && mapAsync(transport, (transport) => { - const opts: ConnectionOpts = { transport, client: this.matrixRTCSession.room.client, @@ -582,15 +584,16 @@ export class CallViewModel extends ViewModel { remoteTransports$: this.remoteTransports$, }; return { - connection: new PublishConnection( - opts, - this.mediaDevices, - this.muteStates, - this.e2eeLivekitOptions(), - this.scope.behavior(this.trackProcessorState$), - ), - transport, - }}), + connection: new PublishConnection( + opts, + this.mediaDevices, + this.muteStates, + this.e2eeLivekitOptions(), + this.scope.behavior(this.trackProcessorState$), + ), + transport, + }; + }), ), ), ); @@ -605,14 +608,14 @@ export class CallViewModel extends ViewModel { this.localConnection$.pipe( switchMap((c) => c?.state === "ready" - // TODO mapping to ConnectionState for compatibility, but we should use the full state? - ? c.value.focusedConnectionState$.pipe( - map((s) => { - if (s.state === "ConnectedToLkRoom") return s.connectionState; - return ConnectionState.Disconnected - }), - distinctUntilChanged(), - ) + ? // TODO mapping to ConnectionState for compatibility, but we should use the full state? + c.value.focusedConnectionState$.pipe( + map((s) => { + if (s.state === "ConnectedToLkRoom") return s.connectionState; + return ConnectionState.Disconnected; + }), + distinctUntilChanged(), + ) : of(ConnectionState.Disconnected), ), ), @@ -659,8 +662,11 @@ export class CallViewModel extends ViewModel { client: this.matrixRTCSession.room.client, scope: this.scope, remoteTransports$: this.remoteTransports$, - } - nextConnection = new RemoteConnection(args, this.e2eeLivekitOptions()); + }; + nextConnection = new RemoteConnection( + args, + this.e2eeLivekitOptions(), + ); } else { logger.log( "SFU remoteConnections$ use prev connection: ", @@ -1952,16 +1958,20 @@ export class CallViewModel extends ViewModel { .pipe(this.scope.bind()) .subscribe(({ start, stop }) => { for (const c of stop) { - logger.info(`Disconnecting from ${c.localTransport.livekit_service_url}`); + logger.info( + `Disconnecting from ${c.localTransport.livekit_service_url}`, + ); c.stop().catch((err) => { // TODO: better error handling logger.error("MuteState: handler error", err); - });; + }); } for (const c of start) { c.start().then( () => - logger.info(`Connected to ${c.localTransport.livekit_service_url}`), + logger.info( + `Connected to ${c.localTransport.livekit_service_url}`, + ), (e) => logger.error( `Failed to start connection to ${c.localTransport.livekit_service_url}`, diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 74a615156..699422707 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -5,21 +5,37 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { afterEach, describe, expect, it, type Mock, type MockedObject, vi } from "vitest"; +import { + afterEach, + describe, + expect, + it, + type Mock, + type MockedObject, + vi, +} from "vitest"; import { BehaviorSubject, of } from "rxjs"; import { ConnectionState, type LocalParticipant, type RemoteParticipant, type Room as LivekitRoom, - RoomEvent, type RoomOptions + RoomEvent, + type RoomOptions, } from "livekit-client"; import fetchMock from "fetch-mock"; import EventEmitter from "events"; import { type IOpenIDToken } from "matrix-js-sdk"; -import type { CallMembership, LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; -import { type ConnectionOpts, type FocusConnectionState, RemoteConnection } from "./Connection.ts"; +import type { + CallMembership, + LivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; +import { + type ConnectionOpts, + type FocusConnectionState, + RemoteConnection, +} from "./Connection.ts"; import { ObservableScope } from "./ObservableScope.ts"; import { type OpenIDClientParts } from "../livekit/openIDSFU.ts"; import { FailToGetOpenIdToken } from "../utils/errors.ts"; @@ -38,28 +54,30 @@ let localParticipantEventEmiter: EventEmitter; let fakeLocalParticipant: MockedObject; let fakeRoomEventEmiter: EventEmitter; -let fakeMembershipsFocusMap$: BehaviorSubject<{ membership: CallMembership; transport: LivekitTransport }[]>; +let fakeMembershipsFocusMap$: BehaviorSubject< + { membership: CallMembership; transport: LivekitTransport }[] +>; const livekitFocus: LivekitTransport = { livekit_alias: "!roomID:example.org", livekit_service_url: "https://matrix-rtc.example.org/livekit/jwt", - type: "livekit" + type: "livekit", }; function setupTest(): void { testScope = new ObservableScope(); client = vi.mocked({ - getOpenIdToken: vi.fn().mockResolvedValue( - { - "access_token": "rYsmGUEwNjKgJYyeNUkZseJN", - "token_type": "Bearer", - "matrix_server_name": "example.org", - "expires_in": 3600 - } - ), - getDeviceId: vi.fn().mockReturnValue("ABCDEF") + getOpenIdToken: vi.fn().mockResolvedValue({ + access_token: "rYsmGUEwNjKgJYyeNUkZseJN", + token_type: "Bearer", + matrix_server_name: "example.org", + expires_in: 3600, + }), + getDeviceId: vi.fn().mockReturnValue("ABCDEF"), } as unknown as OpenIDClientParts); - fakeMembershipsFocusMap$ = new BehaviorSubject<{ membership: CallMembership; transport: LivekitTransport }[]>([]); + fakeMembershipsFocusMap$ = new BehaviorSubject< + { membership: CallMembership; transport: LivekitTransport }[] + >([]); localParticipantEventEmiter = new EventEmitter(); @@ -69,9 +87,15 @@ function setupTest(): void { getTrackPublication: vi.fn().mockReturnValue(undefined), on: localParticipantEventEmiter.on.bind(localParticipantEventEmiter), off: localParticipantEventEmiter.off.bind(localParticipantEventEmiter), - addListener: localParticipantEventEmiter.addListener.bind(localParticipantEventEmiter), - removeListener: localParticipantEventEmiter.removeListener.bind(localParticipantEventEmiter), - removeAllListeners: localParticipantEventEmiter.removeAllListeners.bind(localParticipantEventEmiter) + addListener: localParticipantEventEmiter.addListener.bind( + localParticipantEventEmiter, + ), + removeListener: localParticipantEventEmiter.removeListener.bind( + localParticipantEventEmiter, + ), + removeAllListeners: localParticipantEventEmiter.removeAllListeners.bind( + localParticipantEventEmiter, + ), } as unknown as LocalParticipant); fakeRoomEventEmiter = new EventEmitter(); @@ -84,56 +108,45 @@ function setupTest(): void { on: fakeRoomEventEmiter.on.bind(fakeRoomEventEmiter), off: fakeRoomEventEmiter.off.bind(fakeRoomEventEmiter), addListener: fakeRoomEventEmiter.addListener.bind(fakeRoomEventEmiter), - removeListener: fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), - removeAllListeners: fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), - setE2EEEnabled: vi.fn().mockResolvedValue(undefined) + removeListener: + fakeRoomEventEmiter.removeListener.bind(fakeRoomEventEmiter), + removeAllListeners: + fakeRoomEventEmiter.removeAllListeners.bind(fakeRoomEventEmiter), + setE2EEEnabled: vi.fn().mockResolvedValue(undefined), } as unknown as LivekitRoom); - } function setupRemoteConnection(): RemoteConnection { - const opts: ConnectionOpts = { client: client, transport: livekitFocus, remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom + livekitRoomFactory: () => fakeLivekitRoom, }; - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, - () => { - return { - status: 200, - body: - { - "url": "wss://matrix-rtc.m.localhost/livekit/sfu", - "jwt": "ATOKEN" - } - }; - } - ); + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { + return { + status: 200, + body: { + url: "wss://matrix-rtc.m.localhost/livekit/sfu", + jwt: "ATOKEN", + }, + }; + }); - fakeLivekitRoom - .connect - .mockResolvedValue(undefined); + fakeLivekitRoom.connect.mockResolvedValue(undefined); - return new RemoteConnection( - opts, - undefined - ); + return new RemoteConnection(opts, undefined); } - afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); fetchMock.reset(); }); - describe("Start connection states", () => { - it("start in initialized state", () => { setupTest(); @@ -142,15 +155,13 @@ describe("Start connection states", () => { transport: livekitFocus, remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom + livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new RemoteConnection( - opts, - undefined - ); + const connection = new RemoteConnection(opts, undefined); - expect(connection.focusedConnectionState$.getValue().state) - .toEqual("Initialized"); + expect(connection.focusedConnectionState$.getValue().state).toEqual( + "Initialized", + ); }); it("fail to getOpenId token then error state", async () => { @@ -162,31 +173,27 @@ describe("Start connection states", () => { transport: livekitFocus, remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom + livekitRoomFactory: () => fakeLivekitRoom, }; - - const connection = new RemoteConnection( - opts, - undefined - ); + const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { capturedStates.push(value); }); - const deferred = Promise.withResolvers(); - client.getOpenIdToken.mockImplementation(async (): Promise => { - return await deferred.promise; - }); + client.getOpenIdToken.mockImplementation( + async (): Promise => { + return await deferred.promise; + }, + ); - connection.start() - .catch(() => { - // expected to throw - }); + connection.start().catch(() => { + // expected to throw + }); let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); @@ -199,11 +206,14 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if (capturedState!.state === "FailedToStart") { expect(capturedState!.error.message).toEqual("Something went wrong"); - expect(capturedState!.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + expect(capturedState!.focus.livekit_alias).toEqual( + livekitFocus.livekit_alias, + ); } else { - expect.fail("Expected FailedToStart state but got " + capturedState?.state); + expect.fail( + "Expected FailedToStart state but got " + capturedState?.state, + ); } - }); it("fail to get JWT token and error state", async () => { @@ -215,13 +225,10 @@ describe("Start connection states", () => { transport: livekitFocus, remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom + livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new RemoteConnection( - opts, - undefined - ); + const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { @@ -230,21 +237,17 @@ describe("Start connection states", () => { const deferredSFU = Promise.withResolvers(); // mock the /sfu/get call - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, - async () => { - await deferredSFU.promise; - return { - status: 500, - body: "Internal Server Error" - }; - } - ); - + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, async () => { + await deferredSFU.promise; + return { + status: 500, + body: "Internal Server Error", + }; + }); - connection.start() - .catch(() => { - // expected to throw - }); + connection.start().catch(() => { + // expected to throw + }); let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); @@ -256,15 +259,19 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if (capturedState?.state === "FailedToStart") { - expect(capturedState?.error.message).toContain("SFU Config fetch failed with exception Error"); - expect(capturedState?.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + expect(capturedState?.error.message).toContain( + "SFU Config fetch failed with exception Error", + ); + expect(capturedState?.focus.livekit_alias).toEqual( + livekitFocus.livekit_alias, + ); } else { - expect.fail("Expected FailedToStart state but got " + capturedState?.state); + expect.fail( + "Expected FailedToStart state but got " + capturedState?.state, + ); } - }); - it("fail to connect to livekit error state", async () => { setupTest(); vi.useFakeTimers(); @@ -274,46 +281,36 @@ describe("Start connection states", () => { transport: livekitFocus, remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: () => fakeLivekitRoom + livekitRoomFactory: () => fakeLivekitRoom, }; - const connection = new RemoteConnection( - opts, - undefined - ); + const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; connection.focusedConnectionState$.subscribe((value) => { capturedStates.push(value); }); - const deferredSFU = Promise.withResolvers(); // mock the /sfu/get call - fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, - () => { - return { - status: 200, - body: - { - "url": "wss://matrix-rtc.m.localhost/livekit/sfu", - "jwt": "ATOKEN" - } - }; - } - ); + fetchMock.post(`${livekitFocus.livekit_service_url}/sfu/get`, () => { + return { + status: 200, + body: { + url: "wss://matrix-rtc.m.localhost/livekit/sfu", + jwt: "ATOKEN", + }, + }; + }); - fakeLivekitRoom - .connect - .mockImplementation(async () => { - await deferredSFU.promise; - throw new Error("Failed to connect to livekit"); - }); + fakeLivekitRoom.connect.mockImplementation(async () => { + await deferredSFU.promise; + throw new Error("Failed to connect to livekit"); + }); - connection.start() - .catch(() => { - // expected to throw - }); + connection.start().catch(() => { + // expected to throw + }); let capturedState = capturedStates.pop(); expect(capturedState).toBeDefined(); @@ -326,12 +323,17 @@ describe("Start connection states", () => { capturedState = capturedStates.pop(); if (capturedState && capturedState?.state === "FailedToStart") { - expect(capturedState.error.message).toContain("Failed to connect to livekit"); - expect(capturedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); + expect(capturedState.error.message).toContain( + "Failed to connect to livekit", + ); + expect(capturedState.focus.livekit_alias).toEqual( + livekitFocus.livekit_alias, + ); } else { - expect.fail("Expected FailedToStart state but got " + JSON.stringify(capturedState)); + expect.fail( + "Expected FailedToStart state but got " + JSON.stringify(capturedState), + ); } - }); it("connection states happy path", async () => { @@ -356,7 +358,6 @@ describe("Start connection states", () => { expect(connectingState?.state).toEqual("ConnectingToLkRoom"); const connectedState = capturedState.shift(); expect(connectedState?.state).toEqual("ConnectedToLkRoom"); - }); it("should relay livekit events once connected", async () => { @@ -378,7 +379,7 @@ describe("Start connection states", () => { ConnectionState.SignalReconnecting, ConnectionState.Connecting, ConnectionState.Connected, - ConnectionState.Reconnecting + ConnectionState.Reconnecting, ]; for (const state of states) { fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state); @@ -387,12 +388,18 @@ describe("Start connection states", () => { for (const state of states) { const s = capturedState.shift(); expect(s?.state).toEqual("ConnectedToLkRoom"); - const connectedState = s as FocusConnectionState & { state: "ConnectedToLkRoom" }; + const connectedState = s as FocusConnectionState & { + state: "ConnectedToLkRoom"; + }; expect(connectedState.connectionState).toEqual(state); // should always have the focus info - expect(connectedState.focus.livekit_alias).toEqual(livekitFocus.livekit_alias); - expect(connectedState.focus.livekit_service_url).toEqual(livekitFocus.livekit_service_url); + expect(connectedState.focus.livekit_alias).toEqual( + livekitFocus.livekit_alias, + ); + expect(connectedState.focus.livekit_service_url).toEqual( + livekitFocus.livekit_service_url, + ); } // If the state is not ConnectedToLkRoom, no events should be relayed anymore @@ -403,10 +410,8 @@ describe("Start connection states", () => { } expect(capturedState.length).toEqual(0); - }); - it("shutting down the scope should stop the connection", async () => { setupTest(); vi.useFakeTimers(); @@ -423,7 +428,6 @@ describe("Start connection states", () => { const stopSpy = vi.spyOn(connection, "stop"); testScope.end(); - expect(stopSpy).toHaveBeenCalled(); expect(fakeLivekitRoom.disconnect).toHaveBeenCalled(); @@ -437,26 +441,22 @@ describe("Start connection states", () => { expect(capturedState.length).toEqual(0); }); - }); - function fakeRemoteLivekitParticipant(id: string): RemoteParticipant { return vi.mocked({ - identity: id + identity: id, } as unknown as RemoteParticipant); } function fakeRtcMemberShip(userId: string, deviceId: string): CallMembership { return vi.mocked({ sender: userId, - deviceId: deviceId + deviceId: deviceId, } as unknown as CallMembership); } describe("Publishing participants observations", () => { - - it("should emit the list of publishing participants", async () => { setupTest(); @@ -464,13 +464,24 @@ describe("Publishing participants observations", () => { const bobIsAPublisher = Promise.withResolvers(); const danIsAPublisher = Promise.withResolvers(); - const observedPublishers: { participant: RemoteParticipant; membership: CallMembership }[][] = []; + const observedPublishers: { + participant: RemoteParticipant; + membership: CallMembership; + }[][] = []; connection.publishingParticipants$.subscribe((publishers) => { observedPublishers.push(publishers); - if (publishers.some((p) => p.participant.identity === "@bob:example.org:DEV111")) { + if ( + publishers.some( + (p) => p.participant.identity === "@bob:example.org:DEV111", + ) + ) { bobIsAPublisher.resolve(); } - if (publishers.some((p) => p.participant.identity === "@dan:example.org:DEV333")) { + if ( + publishers.some( + (p) => p.participant.identity === "@dan:example.org:DEV333", + ) + ) { danIsAPublisher.resolve(); } }); @@ -482,14 +493,13 @@ describe("Publishing participants observations", () => { fakeRemoteLivekitParticipant("@alice:example.org:DEV000"), fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), fakeRemoteLivekitParticipant("@carol:example.org:DEV222"), - fakeRemoteLivekitParticipant("@dan:example.org:DEV333") + fakeRemoteLivekitParticipant("@dan:example.org:DEV333"), ]; // Let's simulate 3 members on the livekitRoom - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") - .mockReturnValue( - new Map(participants.map((p) => [p.identity, p])) - ); + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( + new Map(participants.map((p) => [p.identity, p])), + ); for (const participant of participants) { fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); @@ -498,20 +508,27 @@ describe("Publishing participants observations", () => { // At this point there should be no publishers expect(observedPublishers.pop()!.length).toEqual(0); - const otherFocus: LivekitTransport = { livekit_alias: "!roomID:example.org", livekit_service_url: "https://other-matrix-rtc.example.org/livekit/jwt", - type: "livekit" + type: "livekit", }; - const rtcMemberships = [ // Say bob is on the same focus - { membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), transport: livekitFocus }, + { + membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), + transport: livekitFocus, + }, // Alice and carol is on a different focus - { membership: fakeRtcMemberShip("@alice:example.org", "DEV000"), transport: otherFocus }, - { membership: fakeRtcMemberShip("@carol:example.org", "DEV222"), transport: otherFocus } + { + membership: fakeRtcMemberShip("@alice:example.org", "DEV000"), + transport: otherFocus, + }, + { + membership: fakeRtcMemberShip("@carol:example.org", "DEV222"), + transport: otherFocus, + }, // NO DAVE YET ]; // signal this change in rtc memberships @@ -521,53 +538,74 @@ describe("Publishing participants observations", () => { await bobIsAPublisher.promise; const publishers = observedPublishers.pop(); expect(publishers?.length).toEqual(1); - expect(publishers?.[0].participant.identity).toEqual("@bob:example.org:DEV111"); + expect(publishers?.[0].participant.identity).toEqual( + "@bob:example.org:DEV111", + ); // Now let's make dan join the rtc memberships - rtcMemberships - .push({ membership: fakeRtcMemberShip("@dan:example.org", "DEV333"), transport: livekitFocus }); + rtcMemberships.push({ + membership: fakeRtcMemberShip("@dan:example.org", "DEV333"), + transport: livekitFocus, + }); fakeMembershipsFocusMap$.next(rtcMemberships); // We should have bob and dan has publishers now await danIsAPublisher.promise; const twoPublishers = observedPublishers.pop(); expect(twoPublishers?.length).toEqual(2); - expect(twoPublishers?.some((p) => p.participant.identity === "@bob:example.org:DEV111")).toBeTruthy(); - expect(twoPublishers?.some((p) => p.participant.identity === "@dan:example.org:DEV333")).toBeTruthy(); + expect( + twoPublishers?.some( + (p) => p.participant.identity === "@bob:example.org:DEV111", + ), + ).toBeTruthy(); + expect( + twoPublishers?.some( + (p) => p.participant.identity === "@dan:example.org:DEV333", + ), + ).toBeTruthy(); // Now let's make bob leave the livekit room - participants = participants.filter((p) => p.identity !== "@bob:example.org:DEV111"); - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") - .mockReturnValue( - new Map(participants.map((p) => [p.identity, p])) - ); - fakeRoomEventEmiter.emit(RoomEvent.ParticipantDisconnected, fakeRemoteLivekitParticipant("@bob:example.org:DEV111")); + participants = participants.filter( + (p) => p.identity !== "@bob:example.org:DEV111", + ); + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( + new Map(participants.map((p) => [p.identity, p])), + ); + fakeRoomEventEmiter.emit( + RoomEvent.ParticipantDisconnected, + fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + ); const updatedPublishers = observedPublishers.pop(); expect(updatedPublishers?.length).toEqual(1); - expect(updatedPublishers?.some((p) => p.participant.identity === "@dan:example.org:DEV333")).toBeTruthy(); + expect( + updatedPublishers?.some( + (p) => p.participant.identity === "@dan:example.org:DEV333", + ), + ).toBeTruthy(); }); - it("should be scoped to parent scope", (): void => { setupTest(); const connection = setupRemoteConnection(); - let observedPublishers: { participant: RemoteParticipant; membership: CallMembership }[][] = []; + let observedPublishers: { + participant: RemoteParticipant; + membership: CallMembership; + }[][] = []; connection.publishingParticipants$.subscribe((publishers) => { observedPublishers.push(publishers); }); let participants: RemoteParticipant[] = [ - fakeRemoteLivekitParticipant("@bob:example.org:DEV111") + fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), ]; // Let's simulate 3 members on the livekitRoom - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") - .mockReturnValue( - new Map(participants.map((p) => [p.identity, p])) - ); + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( + new Map(participants.map((p) => [p.identity, p])), + ); for (const participant of participants) { fakeRoomEventEmiter.emit(RoomEvent.ParticipantConnected, participant); @@ -578,7 +616,10 @@ describe("Publishing participants observations", () => { const rtcMemberships = [ // Say bob is on the same focus - { membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), transport: livekitFocus } + { + membership: fakeRtcMemberShip("@bob:example.org", "DEV111"), + transport: livekitFocus, + }, ]; // signal this change in rtc memberships fakeMembershipsFocusMap$.next(rtcMemberships); @@ -586,27 +627,31 @@ describe("Publishing participants observations", () => { // We should have bob has a publisher now const publishers = observedPublishers.pop(); expect(publishers?.length).toEqual(1); - expect(publishers?.[0].participant.identity).toEqual("@bob:example.org:DEV111"); + expect(publishers?.[0].participant.identity).toEqual( + "@bob:example.org:DEV111", + ); // end the parent scope testScope.end(); observedPublishers = []; // SHOULD NOT emit any more publishers as the scope is ended - participants = participants.filter((p) => p.identity !== "@bob:example.org:DEV111"); - vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get") - .mockReturnValue( - new Map(participants.map((p) => [p.identity, p])) - ); - fakeRoomEventEmiter.emit(RoomEvent.ParticipantDisconnected, fakeRemoteLivekitParticipant("@bob:example.org:DEV111")); + participants = participants.filter( + (p) => p.identity !== "@bob:example.org:DEV111", + ); + vi.spyOn(fakeLivekitRoom, "remoteParticipants", "get").mockReturnValue( + new Map(participants.map((p) => [p.identity, p])), + ); + fakeRoomEventEmiter.emit( + RoomEvent.ParticipantDisconnected, + fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), + ); expect(observedPublishers.length).toEqual(0); }); }); - describe("PublishConnection", () => { - // let fakeBlurProcessor: ProcessorWrapper; let roomFactoryMock: Mock<() => LivekitRoom>; let muteStates: MockedObject; @@ -616,7 +661,6 @@ describe("PublishConnection", () => { roomFactoryMock = vi.fn().mockReturnValue(fakeLivekitRoom); - muteStates = mockMuteStates(); // fakeBlurProcessor = vi.mocked>({ @@ -626,20 +670,15 @@ describe("PublishConnection", () => { // getOptions: vi.fn().mockReturnValue({ strength: 0.5 }), // isRunning: vi.fn().mockReturnValue(false) // }); - - } - describe("Livekit room creation", () => { - - function createSetup(): void { setUpPublishConnection(); const fakeTrackProcessorSubject$ = new BehaviorSubject({ supported: true, - processor: undefined + processor: undefined, }); const opts: ConnectionOpts = { @@ -647,28 +686,25 @@ describe("PublishConnection", () => { transport: livekitFocus, remoteTransports$: fakeMembershipsFocusMap$, scope: testScope, - livekitRoomFactory: roomFactoryMock + livekitRoomFactory: roomFactoryMock, }; const audioInput = { available$: of(new Map([["mic1", { id: "mic1" }]])), selected$: new BehaviorSubject({ id: "mic1" }), - select(): void { - } + select(): void {}, }; const videoInput = { available$: of(new Map([["cam1", { id: "cam1" }]])), selected$: new BehaviorSubject({ id: "cam1" }), - select(): void { - } + select(): void {}, }; const audioOutput = { available$: of(new Map([["speaker", { id: "speaker" }]])), selected$: new BehaviorSubject({ id: "speaker" }), - select(): void { - } + select(): void {}, }; // TODO understand what is wrong with our mocking that requires ts-expect-error @@ -678,7 +714,7 @@ describe("PublishConnection", () => { // @ts-expect-error Mocking only videoInput, // @ts-expect-error Mocking only - audioOutput + audioOutput, }); new PublishConnection( @@ -686,18 +722,17 @@ describe("PublishConnection", () => { fakeDevices, muteStates, undefined, - fakeTrackProcessorSubject$ + fakeTrackProcessorSubject$, ); - } it("should create room with proper initial audio and video settings", () => { - createSetup(); expect(roomFactoryMock).toHaveBeenCalled(); - const lastCallArgs = roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1]; + const lastCallArgs = + roomFactoryMock.mock.calls[roomFactoryMock.mock.calls.length - 1]; const roomOptions = lastCallArgs.pop() as unknown as RoomOptions; expect(roomOptions).toBeDefined(); @@ -705,7 +740,6 @@ describe("PublishConnection", () => { expect(roomOptions!.videoCaptureDefaults?.deviceId).toEqual("cam1"); expect(roomOptions!.audioCaptureDefaults?.deviceId).toEqual("mic1"); expect(roomOptions!.audioOutput?.deviceId).toEqual("speaker"); - }); it("respect controlledAudioDevices", () => { @@ -719,7 +753,6 @@ describe("PublishConnection", () => { // }) // }; // }); - }); }); }); diff --git a/src/state/Connection.ts b/src/state/Connection.ts index 42423938c..e5e108b7d 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -5,12 +5,27 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { connectedParticipantsObserver, connectionStateObserver } from "@livekit/components-core"; -import { type ConnectionState, type E2EEOptions, Room as LivekitRoom, type RoomOptions } from "livekit-client"; -import { type CallMembership, type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc"; +import { + connectedParticipantsObserver, + connectionStateObserver, +} from "@livekit/components-core"; +import { + type ConnectionState, + type E2EEOptions, + Room as LivekitRoom, + type RoomOptions, +} from "livekit-client"; +import { + type CallMembership, + type LivekitTransport, +} from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, combineLatest } from "rxjs"; -import { getSFUConfigWithOpenID, type OpenIDClientParts, type SFUConfig } from "../livekit/openIDSFU"; +import { + getSFUConfigWithOpenID, + type OpenIDClientParts, + type SFUConfig, +} from "../livekit/openIDSFU"; import { type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; import { defaultLiveKitOptions } from "../livekit/options"; @@ -23,20 +38,26 @@ export interface ConnectionOpts { /** The observable scope to use for this connection. */ scope: ObservableScope; /** An observable of the current RTC call memberships and their associated focus. */ - remoteTransports$: Behavior<{ membership: CallMembership; transport: LivekitTransport }[]>; + remoteTransports$: Behavior< + { membership: CallMembership; transport: LivekitTransport }[] + >; /** Optional factory to create the Livekit room, mainly for testing purposes. */ livekitRoomFactory?: (options?: RoomOptions) => LivekitRoom; } export type FocusConnectionState = - | { state: 'Initialized' } - | { state: 'FetchingConfig', focus: LivekitTransport } - | { state: 'ConnectingToLkRoom', focus: LivekitTransport } - | { state: 'PublishingTracks', focus: LivekitTransport } - | { state: 'FailedToStart', error: Error, focus: LivekitTransport } - | { state: 'ConnectedToLkRoom', connectionState: ConnectionState, focus: LivekitTransport } - | { state: 'Stopped', focus: LivekitTransport }; + | { state: "Initialized" } + | { state: "FetchingConfig"; focus: LivekitTransport } + | { state: "ConnectingToLkRoom"; focus: LivekitTransport } + | { state: "PublishingTracks"; focus: LivekitTransport } + | { state: "FailedToStart"; error: Error; focus: LivekitTransport } + | { + state: "ConnectedToLkRoom"; + connectionState: ConnectionState; + focus: LivekitTransport; + } + | { state: "Stopped"; focus: LivekitTransport }; /** * A connection to a Matrix RTC LiveKit backend. @@ -44,10 +65,9 @@ export type FocusConnectionState = * Expose observables for participants and connection state. */ export class Connection { - // Private Behavior - private readonly _focusedConnectionState$ - = new BehaviorSubject({ state: 'Initialized' }); + private readonly _focusedConnectionState$ = + new BehaviorSubject({ state: "Initialized" }); /** * The current state of the connection to the focus server. @@ -71,31 +91,44 @@ export class Connection { public async start(): Promise { this.stopped = false; try { - this._focusedConnectionState$.next({ state: 'FetchingConfig', focus: this.localTransport }); + this._focusedConnectionState$.next({ + state: "FetchingConfig", + focus: this.localTransport, + }); // TODO could this be loaded earlier to save time? const { url, jwt } = await this.getSFUConfigWithOpenID(); // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; - this._focusedConnectionState$.next({ state: 'ConnectingToLkRoom', focus: this.localTransport }); + this._focusedConnectionState$.next({ + state: "ConnectingToLkRoom", + focus: this.localTransport, + }); await this.livekitRoom.connect(url, jwt); // If we were stopped while connecting, don't proceed to update state. if (this.stopped) return; - this._focusedConnectionState$.next({ state: 'ConnectedToLkRoom', focus: this.localTransport, connectionState: this.livekitRoom.state }); + this._focusedConnectionState$.next({ + state: "ConnectedToLkRoom", + focus: this.localTransport, + connectionState: this.livekitRoom.state, + }); } catch (error) { - this._focusedConnectionState$.next({ state: 'FailedToStart', error: error instanceof Error ? error : new Error(`${error}`), focus: this.localTransport }); + this._focusedConnectionState$.next({ + state: "FailedToStart", + error: error instanceof Error ? error : new Error(`${error}`), + focus: this.localTransport, + }); throw error; } } - protected async getSFUConfigWithOpenID(): Promise { return await getSFUConfigWithOpenID( this.client, this.localTransport.livekit_service_url, - this.localTransport.livekit_alias - ) + this.localTransport.livekit_alias, + ); } /** * Stops the connection. @@ -106,11 +139,13 @@ export class Connection { public async stop(): Promise { if (this.stopped) return; await this.livekitRoom.disconnect(); - this._focusedConnectionState$.next({ state: 'Stopped', focus: this.localTransport }); + this._focusedConnectionState$.next({ + state: "Stopped", + focus: this.localTransport, + }); this.stopped = true; } - /** * An observable of the participants that are publishing on this connection. * This is derived from `participantsIncludingSubscribers$` and `membershipsFocusMap$`. @@ -135,20 +170,20 @@ export class Connection { public readonly livekitRoom: LivekitRoom, opts: ConnectionOpts, ) { - const { transport, client, scope, remoteTransports$ } = - opts; + const { transport, client, scope, remoteTransports$ } = opts; - this.livekitRoom = livekitRoom + this.livekitRoom = livekitRoom; this.localTransport = transport; this.client = client; this.focusedConnectionState$ = scope.behavior( - this._focusedConnectionState$, { state: 'Initialized' } + this._focusedConnectionState$, + { state: "Initialized" }, ); const participantsIncludingSubscribers$ = scope.behavior( connectedParticipantsObserver(this.livekitRoom), - [] + [], ); this.publishingParticipants$ = scope.behavior( @@ -161,7 +196,7 @@ export class Connection { transport.livekit_service_url === this.localTransport.livekit_service_url ? [membership] - : [] + : [], ) // Pair with their associated LiveKit participant (if any) // Uses flatMap to filter out memberships with no associated rtc participant ([]) @@ -171,18 +206,22 @@ export class Connection { return participant ? [{ participant, membership }] : []; }), ), - [] + [], ); - scope.behavior( - connectionStateObserver(this.livekitRoom) - ).subscribe((connectionState) => { - const current = this._focusedConnectionState$.value; - // Only update the state if we are already connected to the LiveKit room. - if (current.state === 'ConnectedToLkRoom') { - this._focusedConnectionState$.next({ state: 'ConnectedToLkRoom', connectionState, focus: current.focus }); - } - }); + scope + .behavior(connectionStateObserver(this.livekitRoom)) + .subscribe((connectionState) => { + const current = this._focusedConnectionState$.value; + // Only update the state if we are already connected to the LiveKit room. + if (current.state === "ConnectedToLkRoom") { + this._focusedConnectionState$.next({ + state: "ConnectedToLkRoom", + connectionState, + focus: current.focus, + }); + } + }); scope.onEnd(() => void this.stop()); } @@ -195,17 +234,21 @@ export class Connection { * It does not publish any local tracks. */ export class RemoteConnection extends Connection { - /** * Creates a new remote connection to a matrix RTC LiveKit backend. * @param opts * @param sharedE2eeOption - The shared E2EE options to use for the connection. */ - public constructor(opts: ConnectionOpts, sharedE2eeOption: E2EEOptions | undefined) { - const factory = opts.livekitRoomFactory ?? ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); + public constructor( + opts: ConnectionOpts, + sharedE2eeOption: E2EEOptions | undefined, + ) { + const factory = + opts.livekitRoomFactory ?? + ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); const livekitRoom = factory({ ...defaultLiveKitOptions, - e2ee: sharedE2eeOption + e2ee: sharedE2eeOption, }); super(livekitRoom, opts); } diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts index 8381c0927..c35c71e46 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/PublishConnection.ts @@ -10,15 +10,24 @@ import { LocalVideoTrack, Room as LivekitRoom, type RoomOptions, - Track + Track, } from "livekit-client"; -import { map, NEVER, type Observable, type Subscription, switchMap } from "rxjs"; +import { + map, + NEVER, + type Observable, + type Subscription, + switchMap, +} from "rxjs"; import { logger } from "matrix-js-sdk/lib/logger"; import type { Behavior } from "./Behavior.ts"; import type { MediaDevices, SelectedDevice } from "./MediaDevices.ts"; import type { MuteStates } from "./MuteStates.ts"; -import { type ProcessorState, trackProcessorSync } from "../livekit/TrackProcessorContext.tsx"; +import { + type ProcessorState, + trackProcessorSync, +} from "../livekit/TrackProcessorContext.tsx"; import { getUrlParams } from "../UrlParams.ts"; import { defaultLiveKitOptions } from "../livekit/options.ts"; import { getValue } from "../utils/observable.ts"; @@ -31,7 +40,6 @@ import { type ObservableScope } from "./ObservableScope.ts"; * This connection will publish the local user's audio and video tracks. */ export class PublishConnection extends Connection { - /** * Creates a new PublishConnection. * @param args - The connection options. {@link ConnectionOpts} @@ -45,15 +53,22 @@ export class PublishConnection extends Connection { devices: MediaDevices, private readonly muteStates: MuteStates, e2eeLivekitOptions: E2EEOptions | undefined, - trackerProcessorState$: Behavior + trackerProcessorState$: Behavior, ) { const { scope } = args; logger.info("[LivekitRoom] Create LiveKit room"); const { controlledAudioDevices } = getUrlParams(); - const factory = args.livekitRoomFactory ?? ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); + const factory = + args.livekitRoomFactory ?? + ((options: RoomOptions): LivekitRoom => new LivekitRoom(options)); const room = factory( - generateRoomOption(devices, trackerProcessorState$.value, controlledAudioDevices, e2eeLivekitOptions) + generateRoomOption( + devices, + trackerProcessorState$.value, + controlledAudioDevices, + e2eeLivekitOptions, + ), ); room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => { logger.error("Failed to set E2EE enabled on room", e); @@ -83,14 +98,14 @@ export class PublishConnection extends Connection { public async start(): Promise { this.stopped = false; - await super.start() + await super.start(); if (this.stopped) return; // TODO this can throw errors? It will also prompt for permissions if not already granted const tracks = await this.livekitRoom.localParticipant.createTracks({ audio: this.muteStates.audio.enabled$.value, - video: this.muteStates.video.enabled$.value + video: this.muteStates.video.enabled$.value, }); if (this.stopped) return; for (const track of tracks) { @@ -100,7 +115,7 @@ export class PublishConnection extends Connection { if (this.stopped) return; // TODO: check if the connection is still active? and break the loop if not? } - }; + } /// Private methods @@ -112,17 +127,19 @@ export class PublishConnection extends Connection { // in the LocalParticipant object for the track object and there's not a nice // way to do that generically. There is usually no OS-level default video capture // device anyway, and audio outputs work differently. - private workaroundRestartAudioInputTrackChrome(devices: MediaDevices, scope: ObservableScope): void { - + private workaroundRestartAudioInputTrackChrome( + devices: MediaDevices, + scope: ObservableScope, + ): void { devices.audioInput.selected$ .pipe( switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER), - scope.bind() + scope.bind(), ) .subscribe(() => { if (this.livekitRoom.state != ConnectionState.Connected) return; const activeMicTrack = Array.from( - this.livekitRoom.localParticipant.audioTrackPublications.values() + this.livekitRoom.localParticipant.audioTrackPublications.values(), ).find((d) => d.source === Track.Source.Microphone)?.track; if ( @@ -147,11 +164,15 @@ export class PublishConnection extends Connection { }); } -// Observe changes in the selected media devices and update the LiveKit room accordingly. - private observeMediaDevices(scope: ObservableScope, devices: MediaDevices, controlledAudioDevices: boolean):void { + // Observe changes in the selected media devices and update the LiveKit room accordingly. + private observeMediaDevices( + scope: ObservableScope, + devices: MediaDevices, + controlledAudioDevices: boolean, + ): void { const syncDevice = ( kind: MediaDeviceKind, - selected$: Observable + selected$: Observable, ): Subscription => selected$.pipe(scope.bind()).subscribe((device) => { if (this.livekitRoom.state != ConnectionState.Connected) return; @@ -160,7 +181,7 @@ export class PublishConnection extends Connection { "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :", this.livekitRoom.getActiveDevice(kind), " !== ", - device?.id + device?.id, ); if ( device !== undefined && @@ -169,7 +190,7 @@ export class PublishConnection extends Connection { this.livekitRoom .switchActiveDevice(kind, device.id) .catch((e) => - logger.error(`Failed to sync ${kind} device with LiveKit`, e) + logger.error(`Failed to sync ${kind} device with LiveKit`, e), ); } }); @@ -208,21 +229,23 @@ export class PublishConnection extends Connection { }); } - private observeTrackProcessors(scope: ObservableScope, room: LivekitRoom, trackerProcessorState$: Behavior): void { + private observeTrackProcessors( + scope: ObservableScope, + room: LivekitRoom, + trackerProcessorState$: Behavior, + ): void { const track$ = scope.behavior( observeTrackReference$(room.localParticipant, Track.Source.Camera).pipe( map((trackRef) => { const track = trackRef?.publication?.track; return track instanceof LocalVideoTrack ? track : null; - }) - ) + }), + ), ); trackProcessorSync(track$, trackerProcessorState$); } - } - // Generate the initial LiveKit RoomOptions based on the current media devices and processor state. function generateRoomOption( devices: MediaDevices, @@ -235,11 +258,11 @@ function generateRoomOption( videoCaptureDefaults: { ...defaultLiveKitOptions.videoCaptureDefaults, deviceId: devices.videoInput.selected$.value?.id, - processor: processorState.processor + processor: processorState.processor, }, audioCaptureDefaults: { ...defaultLiveKitOptions.audioCaptureDefaults, - deviceId: devices.audioInput.selected$.value?.id + deviceId: devices.audioInput.selected$.value?.id, }, audioOutput: { // When using controlled audio devices, we don't want to set the @@ -247,8 +270,8 @@ function generateRoomOption( // (also the id does not need to match a browser device id) deviceId: controlledAudioDevices ? undefined - : getValue(devices.audioOutput.selected$)?.id + : getValue(devices.audioOutput.selected$)?.id, }, - e2ee: e2eeLivekitOptions + e2ee: e2eeLivekitOptions, }; } diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index 57be00ef5..3637e8dec 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -19,7 +19,11 @@ import { type ComponentProps } from "react"; import { MediaView } from "./MediaView"; import { EncryptionStatus } from "../state/MediaViewModel"; -import { mockLocalParticipant, mockMatrixRoomMember, mockRtcMembership } from "../utils/test"; +import { + mockLocalParticipant, + mockMatrixRoomMember, + mockRtcMembership, +} from "../utils/test"; describe("MediaView", () => { const participant = mockLocalParticipant({}); diff --git a/src/utils/test.ts b/src/utils/test.ts index b77e63c00..98a2addfe 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -419,7 +419,9 @@ export const deviceStub = { select(): void {}, }; -export function mockMediaDevices(data: Partial): MockedObject { +export function mockMediaDevices( + data: Partial, +): MockedObject { return vi.mocked({ audioInput: deviceStub, audioOutput: deviceStub, From 18ba02c9c26d78d9fb0cd9c67834bccccb78e089 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 7 Oct 2025 16:29:11 +0200 Subject: [PATCH 18/46] knip: remove dead code --- src/room/MuteStates.ts | 11 ++++---- src/useMatrixRTCSessionJoinState.ts | 40 ----------------------------- src/utils/test-fixtures.ts | 3 +-- 3 files changed, 7 insertions(+), 47 deletions(-) delete mode 100644 src/useMatrixRTCSessionJoinState.ts diff --git a/src/room/MuteStates.ts b/src/room/MuteStates.ts index e89d13d99..dfc599e79 100644 --- a/src/room/MuteStates.ts +++ b/src/room/MuteStates.ts @@ -27,11 +27,12 @@ import { ElementWidgetActions, widget } from "../widget"; import { Config } from "../config/Config"; import { useUrlParams } from "../UrlParams"; -/** - * If there already are this many participants in the call, we automatically mute - * the user. - */ -export const MUTE_PARTICIPANT_COUNT = 8; +// /** +// * If there already are this many participants in the call, we automatically mute +// * the user. +// */ +// TODO: multi-sfu dead code? +// export const MUTE_PARTICIPANT_COUNT = 8; interface DeviceAvailable { enabled: boolean; diff --git a/src/useMatrixRTCSessionJoinState.ts b/src/useMatrixRTCSessionJoinState.ts deleted file mode 100644 index 2f6ccf257..000000000 --- a/src/useMatrixRTCSessionJoinState.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { logger } from "matrix-js-sdk/lib/logger"; -import { - type MatrixRTCSession, - MatrixRTCSessionEvent, -} from "matrix-js-sdk/lib/matrixrtc"; -import { TypedEventEmitter } from "matrix-js-sdk"; -import { useCallback, useEffect } from "react"; - -import { useTypedEventEmitterState } from "./useEvents"; - -const dummySession = new TypedEventEmitter(); - -export function useMatrixRTCSessionJoinState( - rtcSession: MatrixRTCSession | undefined, -): boolean { - // React doesn't allow you to run a hook conditionally, so we have to plug in - // a dummy event emitter in case there is no rtcSession yet - const isJoined = useTypedEventEmitterState( - rtcSession ?? dummySession, - MatrixRTCSessionEvent.JoinStateChanged, - useCallback(() => rtcSession?.isJoined() ?? false, [rtcSession]), - ); - - useEffect(() => { - logger.info( - `Session in room ${rtcSession?.room.roomId} changed to ${ - isJoined ? "joined" : "left" - }`, - ); - }, [rtcSession, isJoined]); - - return isJoined; -} diff --git a/src/utils/test-fixtures.ts b/src/utils/test-fixtures.ts index 6a8b641b9..9d93267e0 100644 --- a/src/utils/test-fixtures.ts +++ b/src/utils/test-fixtures.ts @@ -9,7 +9,6 @@ import { mockRtcMembership, mockMatrixRoomMember, mockRemoteParticipant, - mockLocalParticipant, } from "./test"; export const localRtcMember = mockRtcMembership("@carol:example.org", "1111"); @@ -18,7 +17,7 @@ export const localRtcMemberDevice2 = mockRtcMembership( "2222", ); export const local = mockMatrixRoomMember(localRtcMember); -export const localParticipant = mockLocalParticipant({ identity: "" }); +// export const localParticipant = mockLocalParticipant({ identity: "" }); export const localId = `${local.userId}:${localRtcMember.deviceId}`; export const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA"); From 05e7b5a7ffb71a7a916b25632b526a8bf3a8f397 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 7 Oct 2025 17:35:25 +0200 Subject: [PATCH 19/46] fixup MediaView tests --- src/tile/MediaView.test.tsx | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index 3637e8dec..672f33344 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -19,11 +19,7 @@ import { type ComponentProps } from "react"; import { MediaView } from "./MediaView"; import { EncryptionStatus } from "../state/MediaViewModel"; -import { - mockLocalParticipant, - mockMatrixRoomMember, - mockRtcMembership, -} from "../utils/test"; +import { mockLocalParticipant } from "../utils/test"; describe("MediaView", () => { const participant = mockLocalParticipant({}); @@ -49,10 +45,7 @@ describe("MediaView", () => { mirror: false, unencryptedWarning: false, video: trackReference, - member: mockMatrixRoomMember( - mockRtcMembership("@alice:example.org", "CCCC"), - { name: "some name" }, - ), + member: undefined, localParticipant: false, focusable: true, }; From 13fb46644c48735fcf69837439f09bcfbd346210 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 8 Oct 2025 08:50:35 +0200 Subject: [PATCH 20/46] test: Fix mediaView test, ,member is not optional anymore Updated the test because now name will be the userId instead of default display name --- src/tile/MediaView.test.tsx | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index 672f33344..abf293136 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { describe, expect, it, test } from "vitest"; +import { describe, expect, it, test, vi } from "vitest"; import { render, screen } from "@testing-library/react"; import { axe } from "vitest-axe"; import { TooltipProvider } from "@vector-im/compound-web"; @@ -16,6 +16,7 @@ import { import { LocalTrackPublication, Track } from "livekit-client"; import { TrackInfo } from "@livekit/protocol"; import { type ComponentProps } from "react"; +import { type RoomMember } from "matrix-js-sdk"; import { MediaView } from "./MediaView"; import { EncryptionStatus } from "../state/MediaViewModel"; @@ -45,7 +46,11 @@ describe("MediaView", () => { mirror: false, unencryptedWarning: false, video: trackReference, - member: undefined, + member: vi.mocked({ + name: () => "some name", + userId: "@alice:example.com", + getMxcAvatarUrl: vi.fn().mockReturnValue(undefined), + }), localParticipant: false, focusable: true, }; @@ -59,9 +64,9 @@ describe("MediaView", () => { test("neither video nor avatar are shown", () => { render(); expect(screen.queryByTestId("video")).toBeNull(); - expect(screen.queryAllByRole("img", { name: "some name" }).length).toBe( - 0, - ); + expect( + screen.queryAllByRole("img", { name: "@alice:example.com" }).length, + ).toBe(0); }); }); @@ -70,14 +75,18 @@ describe("MediaView", () => { render( , ); - expect(screen.getByRole("img", { name: "some name" })).toBeVisible(); + expect( + screen.getByRole("img", { name: "@alice:example.com" }), + ).toBeVisible(); expect(screen.queryAllByText("Waiting for media...").length).toBe(0); }); it("shows avatar and label for remote user", () => { render( , ); - expect(screen.getByRole("img", { name: "some name" })).toBeVisible(); + expect( + screen.getByRole("img", { name: "@alice:example.com" }), + ).toBeVisible(); expect(screen.getByText("Waiting for media...")).toBeVisible(); }); }); @@ -131,7 +140,9 @@ describe("MediaView", () => { , ); - expect(screen.getByRole("img", { name: "some name" })).toBeVisible(); + expect( + screen.getByRole("img", { name: "@alice:example.com" }), + ).toBeVisible(); expect(screen.getByTestId("video")).not.toBeVisible(); }); }); From f5ea734a5c3dea09f5c7e4cd31339745823b9b48 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 8 Oct 2025 14:29:59 +0200 Subject: [PATCH 21/46] esLint fix --- src/tile/MediaView.test.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tile/MediaView.test.tsx b/src/tile/MediaView.test.tsx index abf293136..c26a4d5f5 100644 --- a/src/tile/MediaView.test.tsx +++ b/src/tile/MediaView.test.tsx @@ -47,10 +47,9 @@ describe("MediaView", () => { unencryptedWarning: false, video: trackReference, member: vi.mocked({ - name: () => "some name", userId: "@alice:example.com", getMxcAvatarUrl: vi.fn().mockReturnValue(undefined), - }), + } as unknown as RoomMember), localParticipant: false, focusable: true, }; From afe004c6e7306582e6eacd3ec7e1c8c96207d9c4 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 8 Oct 2025 14:30:52 +0200 Subject: [PATCH 22/46] Remove un-necessary transport field, already accessible from connection --- src/state/CallViewModel.ts | 50 ++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index f517908f8..e0045d150 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -563,7 +563,7 @@ export class CallViewModel extends ViewModel { /** * The transport over which we should be actively publishing our media. */ - private readonly localTransport$: Behavior | null> = + private readonly localTransport$: Behavior> = this.scope.behavior( this.transports$.pipe( map((transports) => transports?.local ?? null), @@ -571,38 +571,30 @@ export class CallViewModel extends ViewModel { ), ); - private readonly localConnectionAndTransport$ = this.scope.behavior( - this.localTransport$.pipe( - map( - (transport) => - transport && - mapAsync(transport, (transport) => { - const opts: ConnectionOpts = { - transport, - client: this.matrixRTCSession.room.client, - scope: this.scope, - remoteTransports$: this.remoteTransports$, - }; - return { - connection: new PublishConnection( + private readonly localConnection$: Behavior> = + this.scope.behavior( + this.localTransport$.pipe( + map( + (transport) => + transport && + mapAsync(transport, (transport) => { + const opts: ConnectionOpts = { + transport, + client: this.matrixRTCSession.room.client, + scope: this.scope, + remoteTransports$: this.remoteTransports$, + }; + return new PublishConnection( opts, this.mediaDevices, this.muteStates, this.e2eeLivekitOptions(), this.scope.behavior(this.trackProcessorState$), - ), - transport, - }; - }), + ); + }), + ), ), - ), - ); - - private readonly localConnection$ = this.scope.behavior( - this.localConnectionAndTransport$.pipe( - map((value) => value && mapAsync(value, ({ connection }) => connection)), - ), - ); + ); public readonly livekitConnectionState$ = this.scope.behavior( this.localConnection$.pipe( @@ -813,11 +805,11 @@ export class CallViewModel extends ViewModel { }[] >( // TODO: Move this logic into Connection/PublishConnection if possible - this.localConnectionAndTransport$ + this.localConnection$ .pipe( switchMap((values) => { if (values?.state !== "ready") return []; - const localConnection = values.value.connection; + const localConnection = values.value; const memberError = (): never => { throw new Error("No room member for call membership"); }; From 427a8dd644dd86271b8122c763c29e0a71cf08e2 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 8 Oct 2025 14:48:40 +0200 Subject: [PATCH 23/46] test: Fix Audio render tests and added more --- src/livekit/MatrixAudioRenderer.test.tsx | 225 ++++++++++++++--------- src/livekit/MatrixAudioRenderer.tsx | 18 +- src/utils/test.ts | 7 +- 3 files changed, 149 insertions(+), 101 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index 075927327..c1ee6f83a 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -6,21 +6,24 @@ Please see LICENSE in the repository root for full details. */ import { afterEach, beforeEach, expect, it, vi } from "vitest"; -import { render } from "@testing-library/react"; +import { render, type RenderResult } from "@testing-library/react"; import { getTrackReferenceId, type TrackReference, } from "@livekit/components-core"; -import { type RemoteAudioTrack } from "livekit-client"; +import { + type Participant, + type RemoteAudioTrack, + type RemoteParticipant, + type Room, +} from "livekit-client"; import { type ReactNode } from "react"; import { useTracks } from "@livekit/components-react"; -import { of } from "rxjs"; import { testAudioContext } from "../useAudioContext.test"; import * as MediaDevicesContext from "../MediaDevicesContext"; import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer"; import { - mockLivekitRoom, mockMatrixRoomMember, mockMediaDevices, mockRtcMembership, @@ -54,90 +57,148 @@ vi.mock("@livekit/components-react", async (importOriginal) => { }; }); -const tracks = [mockTrack("test:123")]; -vi.mocked(useTracks).mockReturnValue(tracks); - -it("should render for member", () => { - // TODO this is duplicated test setup in all tests - const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); - const carol = mockMatrixRoomMember(localRtcMember); - const p = { - id: "test:123", - participant: undefined, - member: carol, - }; - const livekitRoom = mockLivekitRoom( - {}, - { - remoteParticipants$: of([]), - }, - ); - const { container, queryAllByTestId } = render( +let tracks: TrackReference[] = []; + +/** + * Render the test component with given rtc members and livekit participant identities. + * + * It is possible to have rtc members that are not in livekit (e.g. not yet joined) and vice versa. + * + * @param rtcMembers - Array of active rtc members with userId and deviceId. + * @param livekitParticipantIdentities - Array of livekit participant (that are publishing). + * */ + +function renderTestComponent( + rtcMembers: { userId: string; deviceId: string }[], + livekitParticipantIdentities: ({ id: string; isLocal?: boolean } | string)[], +): RenderResult { + const liveKitParticipants = livekitParticipantIdentities.map((p) => { + const identity = typeof p === "string" ? p : p.id; + const isLocal = typeof p === "string" ? false : (p.isLocal ?? false); + return vi.mocked({ + identity, + isLocal, + } as unknown as RemoteParticipant); + }); + const participants = rtcMembers.map(({ userId, deviceId }) => { + const p = liveKitParticipants.find( + (p) => p.identity === `${userId}:${deviceId}`, + ); + const localRtcMember = mockRtcMembership(userId, deviceId); + const member = mockMatrixRoomMember(localRtcMember); + return { + id: `${userId}:${deviceId}`, + participant: p, + member, + }; + }); + const livekitRoom = vi.mocked({ + remoteParticipants: new Map( + liveKitParticipants.map((p) => [p.identity, p]), + ), + } as unknown as Room); + + tracks = participants + .filter((p) => p.participant) + .map((p) => mockTrack(p.participant!)) as TrackReference[]; + + vi.mocked(useTracks).mockReturnValue(tracks); + return render( , ); +} + +it("should render for member", () => { + const { container, queryAllByTestId } = renderTestComponent( + [{ userId: "@alice", deviceId: "DEV0" }], + ["@alice:DEV0"], + ); expect(container).toBeTruthy(); expect(queryAllByTestId("audio")).toHaveLength(1); }); it("should not render without member", () => { - // const memberships = [ - // { sender: "othermember", deviceId: "123" }, - // ] as CallMembership[]; - const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); - const carol = mockMatrixRoomMember(localRtcMember); - const p = { - id: "test:123", - participant: undefined, - member: carol, - }; - const livekitRoom = mockLivekitRoom( - {}, - { - remoteParticipants$: of([]), - }, - ); - const { container, queryAllByTestId } = render( - - - , + const { container, queryAllByTestId } = renderTestComponent( + [{ userId: "@bob", deviceId: "DEV0" }], + ["@alice:DEV0"], ); expect(container).toBeTruthy(); expect(queryAllByTestId("audio")).toHaveLength(0); }); +const TEST_CASES: { + rtcUsers: { userId: string; deviceId: string }[]; + livekitParticipantIdentities: (string | { id: string; isLocal?: boolean })[]; + expectedAudioTracks: number; +}[] = [ + { + rtcUsers: [ + { userId: "@alice", deviceId: "DEV0" }, + { userId: "@alice", deviceId: "DEV1" }, + { userId: "@bob", deviceId: "DEV0" }, + ], + livekitParticipantIdentities: [ + { id: "@alice:DEV0" }, + "@bob:DEV0", + "@alice:DEV1", + ], + expectedAudioTracks: 3, + }, + // Alice DEV0 is local participant, should not render + { + rtcUsers: [ + { userId: "@alice", deviceId: "DEV0" }, + { userId: "@alice", deviceId: "DEV1" }, + { userId: "@bob", deviceId: "DEV0" }, + ], + livekitParticipantIdentities: [ + { id: "@alice:DEV0", isLocal: true }, + "@bob:DEV0", + "@alice:DEV1", + ], + expectedAudioTracks: 2, + }, + // Charlie is a rtc member but not in livekit + { + rtcUsers: [ + { userId: "@alice", deviceId: "DEV0" }, + { userId: "@bob", deviceId: "DEV0" }, + { userId: "@charlie", deviceId: "DEV0" }, + ], + livekitParticipantIdentities: ["@alice:DEV0", { id: "@bob:DEV0" }], + expectedAudioTracks: 2, + }, + // Charlie is in livekit but not rtc member + { + rtcUsers: [ + { userId: "@alice", deviceId: "DEV0" }, + { userId: "@bob", deviceId: "DEV0" }, + ], + livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@charlie:DEV0"], + expectedAudioTracks: 2, + }, +]; + +TEST_CASES.forEach( + ({ rtcUsers, livekitParticipantIdentities, expectedAudioTracks }, index) => { + it(`should render sound test cases #${index + 1}`, () => { + const { queryAllByTestId } = renderTestComponent( + rtcUsers, + livekitParticipantIdentities, + ); + expect(queryAllByTestId("audio")).toHaveLength(expectedAudioTracks); + }); + }, +); + it("should not setup audioContext gain and pan if there is no need to.", () => { - const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); - const carol = mockMatrixRoomMember(localRtcMember); - const p = { - id: "test:123", - participant: undefined, - member: carol, - }; - const livekitRoom = mockLivekitRoom( - {}, - { - remoteParticipants$: of([]), - }, - ); - render( - - - , - ); + renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]); const audioTrack = tracks[0].publication.track! as RemoteAudioTrack; expect(audioTrack.setAudioContext).toHaveBeenCalledTimes(1); @@ -154,28 +215,8 @@ it("should setup audioContext gain and pan", () => { pan: 1, volume: 0.1, }); - const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC"); - const carol = mockMatrixRoomMember(localRtcMember); - const p = { - id: "test:123", - participant: undefined, - member: carol, - }; - const livekitRoom = mockLivekitRoom( - {}, - { - remoteParticipants$: of([]), - }, - ); - render( - - - , - ); + + renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]); const audioTrack = tracks[0].publication.track! as RemoteAudioTrack; expect(audioTrack.setAudioContext).toHaveBeenCalled(); diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 76c206c7e..24455f703 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -32,6 +32,7 @@ export interface MatrixAudioRendererProps { * This list needs to be composed based on the matrixRTC members so that we do not play audio from users * that are not expected to be in the rtc session. */ + // TODO: Why do we have this structure? looks like we only need the valid/active participants (not the room member or id)? participants: { id: string; // TODO it appears to be optional as per InCallView? but what does that mean here? a rtc member not yet joined in livekit? @@ -66,8 +67,15 @@ export function LivekitRoomAudioRenderer({ participants, muted, }: MatrixAudioRendererProps): ReactNode { - const participantSet = useMemo( - () => new Set(participants.map(({ participant }) => participant)), + // This is the list of valid identities that are allowed to play audio. + // It is derived from the list of matrix rtc members. + const validIdentities = useMemo( + () => + new Set( + participants + .filter(({ participant }) => participant) // filter out participants that are not yet joined in livekit + .map(({ participant }) => participant!.identity), + ), [participants], ); @@ -102,7 +110,7 @@ export function LivekitRoomAudioRenderer({ room: livekitRoom, }, ).filter((ref) => { - const isValid = participantSet?.has(ref.participant); + const isValid = validIdentities.has(ref.participant.identity); if (!isValid && !ref.participant.isLocal) logInvalid(ref.participant.identity); return ( @@ -115,14 +123,14 @@ export function LivekitRoomAudioRenderer({ useEffect(() => { if ( loggedInvalidIdentities.current.size && - tracks.every((t) => participantSet.has(t.participant)) + tracks.every((t) => validIdentities.has(t.participant.identity)) ) { logger.debug( `[MatrixAudioRenderer] All audio tracks from ${url} have a matching matrix call member identity.`, ); loggedInvalidIdentities.current.clear(); } - }, [tracks, participantSet, url]); + }, [tracks, validIdentities, url]); // This component is also (in addition to the "only play audio for connected members" logic above) // responsible for mimicking earpiece audio on iPhones. diff --git a/src/utils/test.ts b/src/utils/test.ts index 98a2addfe..d0e08dd89 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -28,6 +28,7 @@ import { type MembershipManagerEventHandlerMap } from "matrix-js-sdk/lib/matrixr import { type LocalParticipant, type LocalTrackPublication, + type Participant, type RemoteParticipant, type RemoteTrackPublication, type Room as LivekitRoom, @@ -392,11 +393,9 @@ export class MockRTCSession extends TypedEventEmitter< } } -export const mockTrack = (identity: string): TrackReference => +export const mockTrack = (participant: Participant): TrackReference => ({ - participant: { - identity, - }, + participant, publication: { kind: Track.Kind.Audio, source: "mic", From e346c8c148bd6aad302598d57f16f02dc91cf468 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 16:39:27 -0400 Subject: [PATCH 24/46] Re-enable React strict mode --- src/main.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main.tsx b/src/main.tsx index f27b55a41..e6a102c6a 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -11,6 +11,7 @@ Please see LICENSE in the repository root for full details. // dependency references. import "matrix-js-sdk/lib/browser-index"; +import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; import "./index.css"; import { logger } from "matrix-js-sdk/lib/logger"; @@ -59,9 +60,9 @@ if (fatalError !== null) { Initializer.initBeforeReact() .then(() => { root.render( - // - , - // , + + , + , ); }) .catch((e) => { From c96e81bfd375262be9c992cbd1885c7d878f5b8d Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 16:40:06 -0400 Subject: [PATCH 25/46] Simplify type of audio participants exposed from CallViewModel --- src/livekit/MatrixAudioRenderer.test.tsx | 21 ++++-------------- src/livekit/MatrixAudioRenderer.tsx | 27 ++++++++---------------- src/room/InCallView.tsx | 4 ++-- src/state/CallViewModel.ts | 23 ++++++++++++++++++-- 4 files changed, 36 insertions(+), 39 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index c1ee6f83a..9519ccc25 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -23,12 +23,7 @@ import { useTracks } from "@livekit/components-react"; import { testAudioContext } from "../useAudioContext.test"; import * as MediaDevicesContext from "../MediaDevicesContext"; import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer"; -import { - mockMatrixRoomMember, - mockMediaDevices, - mockRtcMembership, - mockTrack, -} from "../utils/test"; +import { mockMediaDevices, mockTrack } from "../utils/test"; export const TestAudioContextConstructor = vi.fn(() => testAudioContext); @@ -80,17 +75,11 @@ function renderTestComponent( isLocal, } as unknown as RemoteParticipant); }); - const participants = rtcMembers.map(({ userId, deviceId }) => { + const participants = rtcMembers.flatMap(({ userId, deviceId }) => { const p = liveKitParticipants.find( (p) => p.identity === `${userId}:${deviceId}`, ); - const localRtcMember = mockRtcMembership(userId, deviceId); - const member = mockMatrixRoomMember(localRtcMember); - return { - id: `${userId}:${deviceId}`, - participant: p, - member, - }; + return p === undefined ? [] : [p]; }); const livekitRoom = vi.mocked({ remoteParticipants: new Map( @@ -98,9 +87,7 @@ function renderTestComponent( ), } as unknown as Room); - tracks = participants - .filter((p) => p.participant) - .map((p) => mockTrack(p.participant!)) as TrackReference[]; + tracks = participants.map((p) => mockTrack(p)); vi.mocked(useTracks).mockReturnValue(tracks); return render( diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index 24455f703..fb1400b43 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -6,7 +6,10 @@ Please see LICENSE in the repository root for full details. */ import { getTrackReferenceId } from "@livekit/components-core"; -import { type Room as LivekitRoom, type Participant } from "livekit-client"; +import { + type RemoteParticipant, + type Room as LivekitRoom, +} from "livekit-client"; import { type RemoteAudioTrack, Track } from "livekit-client"; import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; import { @@ -14,13 +17,13 @@ import { AudioTrack, type AudioTrackProps, } from "@livekit/components-react"; -import { type RoomMember } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/lib/logger"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; import * as controls from "../controls"; import {} from "@livekit/components-core"; + export interface MatrixAudioRendererProps { /** * The service URL of the LiveKit room. @@ -32,13 +35,7 @@ export interface MatrixAudioRendererProps { * This list needs to be composed based on the matrixRTC members so that we do not play audio from users * that are not expected to be in the rtc session. */ - // TODO: Why do we have this structure? looks like we only need the valid/active participants (not the room member or id)? - participants: { - id: string; - // TODO it appears to be optional as per InCallView? but what does that mean here? a rtc member not yet joined in livekit? - participant: Participant | undefined; - member: RoomMember; - }[]; + participants: RemoteParticipant[]; /** * If set to `true`, mutes all audio tracks rendered by the component. * @remarks @@ -48,8 +45,7 @@ export interface MatrixAudioRendererProps { } /** - * The `MatrixAudioRenderer` component is a drop-in solution for adding audio to your LiveKit app. - * It takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible. + * Takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible. * * It also takes care of the earpiece audio configuration for iOS devices. * This is done by using the WebAudio API to create a stereo pan effect that mimics the earpiece audio. @@ -70,12 +66,7 @@ export function LivekitRoomAudioRenderer({ // This is the list of valid identities that are allowed to play audio. // It is derived from the list of matrix rtc members. const validIdentities = useMemo( - () => - new Set( - participants - .filter(({ participant }) => participant) // filter out participants that are not yet joined in livekit - .map(({ participant }) => participant!.identity), - ), + () => new Set(participants.map((p) => p.identity)), [participants], ); @@ -92,7 +83,7 @@ export function LivekitRoomAudioRenderer({ if (loggedInvalidIdentities.current.has(identity)) return; logger.warn( `[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`, - `current members: ${participants.map((p) => p.participant?.identity)}`, + `current members: ${participants.map((p) => p.identity)}`, `track will not get rendered`, ); loggedInvalidIdentities.current.add(identity); diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 8474c2fdc..dacb7eb15 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -286,7 +286,7 @@ export const InCallView: FC = ({ ); const allLivekitRooms = useBehavior(vm.allLivekitRooms$); - const participantsByRoom = useBehavior(vm.participantsByRoom$); + const audioParticipants = useBehavior(vm.audioParticipants$); const participantCount = useBehavior(vm.participantCount$); const reconnecting = useBehavior(vm.reconnecting$); const windowMode = useBehavior(vm.windowMode$); @@ -860,7 +860,7 @@ export const InCallView: FC = ({ ) } - {participantsByRoom.map(({ livekitRoom, url, participants }) => ( + {audioParticipants.map(({ livekitRoom, url, participants }) => ( + data.map(({ livekitRoom, url, participants }) => ({ + livekitRoom, + url, + participants: participants.flatMap(({ participant }) => + participant instanceof RemoteParticipant ? [participant] : [], + ), + })), + ), + ), + ); + /** * Displaynames for each member of the call. This will disambiguate * any displaynames that clashes with another member. Only members From 5da780ed301bc6e259ba9fb52c7b92b3aef88525 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 16:43:25 -0400 Subject: [PATCH 26/46] Remove dead MuteStates file It's been replaced by a refactored RxJS version living in src/state. --- src/room/MuteStates.test.tsx | 4 + src/room/MuteStates.ts | 179 ----------------------------------- 2 files changed, 4 insertions(+), 179 deletions(-) delete mode 100644 src/room/MuteStates.ts diff --git a/src/room/MuteStates.test.tsx b/src/room/MuteStates.test.tsx index eb08217d4..d34f4d391 100644 --- a/src/room/MuteStates.test.tsx +++ b/src/room/MuteStates.test.tsx @@ -5,6 +5,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ +// TODO-MULTI-SFU: These tests need to be ported to the new MuteStates class. +/* + import { afterAll, afterEach, @@ -321,3 +324,4 @@ describe("useMuteStates in VITE_PACKAGE='embedded' (widget) mode", () => { expect(screen.getByTestId("video-enabled").textContent).toBe("true"); }); }); +*/ diff --git a/src/room/MuteStates.ts b/src/room/MuteStates.ts deleted file mode 100644 index dfc599e79..000000000 --- a/src/room/MuteStates.ts +++ /dev/null @@ -1,179 +0,0 @@ -/* -Copyright 2023, 2024 New Vector Ltd. - -SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial -Please see LICENSE in the repository root for full details. -*/ - -import { - type Dispatch, - type SetStateAction, - useCallback, - useEffect, - useMemo, -} from "react"; -import { type IWidgetApiRequest } from "matrix-widget-api"; -import { logger } from "matrix-js-sdk/lib/logger"; -import { useObservableEagerState } from "observable-hooks"; - -import { - type DeviceLabel, - type SelectedDevice, - type MediaDevice, -} from "../state/MediaDevices"; -import { useIsEarpiece, useMediaDevices } from "../MediaDevicesContext"; -import { useReactiveState } from "../useReactiveState"; -import { ElementWidgetActions, widget } from "../widget"; -import { Config } from "../config/Config"; -import { useUrlParams } from "../UrlParams"; - -// /** -// * If there already are this many participants in the call, we automatically mute -// * the user. -// */ -// TODO: multi-sfu dead code? -// export const MUTE_PARTICIPANT_COUNT = 8; - -interface DeviceAvailable { - enabled: boolean; - setEnabled: Dispatch>; -} - -interface DeviceUnavailable { - enabled: false; - setEnabled: null; -} - -const deviceUnavailable: DeviceUnavailable = { - enabled: false, - setEnabled: null, -}; - -type MuteState = DeviceAvailable | DeviceUnavailable; - -export interface MuteStates { - audio: MuteState; - video: MuteState; -} - -function useMuteState( - device: MediaDevice, - enabledByDefault: () => boolean, - forceUnavailable: boolean = false, -): MuteState { - const available = useObservableEagerState(device.available$); - const [enabled, setEnabled] = useReactiveState( - // Determine the default value once devices are actually connected - (prev) => prev ?? (available.size > 0 ? enabledByDefault() : undefined), - [available.size], - ); - return useMemo( - () => - available.size === 0 || forceUnavailable - ? deviceUnavailable - : { - enabled: enabled ?? false, - setEnabled: setEnabled as Dispatch>, - }, - [available.size, enabled, forceUnavailable, setEnabled], - ); -} - -export function useMuteStates(isJoined: boolean): MuteStates { - const devices = useMediaDevices(); - - const { skipLobby, defaultAudioEnabled, defaultVideoEnabled } = - useUrlParams(); - - const audio = useMuteState( - devices.audioInput, - () => - (defaultAudioEnabled ?? Config.get().media_devices.enable_audio) && - allowJoinUnmuted(skipLobby, isJoined), - ); - useEffect(() => { - // If audio is enabled, we need to request the device names again, - // because iOS will not be able to switch to the correct device after un-muting. - // This is one of the main changes that makes iOS work with bluetooth audio devices. - if (audio.enabled) { - devices.requestDeviceNames(); - } - }, [audio.enabled, devices]); - const isEarpiece = useIsEarpiece(); - const video = useMuteState( - devices.videoInput, - () => - (defaultVideoEnabled ?? Config.get().media_devices.enable_video) && - allowJoinUnmuted(skipLobby, isJoined), - isEarpiece, // Force video to be unavailable if using earpiece - ); - - useEffect(() => { - widget?.api.transport - .send(ElementWidgetActions.DeviceMute, { - audio_enabled: audio.enabled, - video_enabled: video.enabled, - }) - .catch((e) => - logger.warn("Could not send DeviceMute action to widget", e), - ); - }, [audio, video]); - - const onMuteStateChangeRequest = useCallback( - (ev: CustomEvent) => { - // First copy the current state into our new state. - const newState = { - audio_enabled: audio.enabled, - video_enabled: video.enabled, - }; - // Update new state if there are any requested changes from the widget action - // in `ev.detail.data`. - if ( - ev.detail.data.audio_enabled != null && - typeof ev.detail.data.audio_enabled === "boolean" - ) { - audio.setEnabled?.(ev.detail.data.audio_enabled); - newState.audio_enabled = ev.detail.data.audio_enabled; - } - if ( - ev.detail.data.video_enabled != null && - typeof ev.detail.data.video_enabled === "boolean" - ) { - video.setEnabled?.(ev.detail.data.video_enabled); - newState.video_enabled = ev.detail.data.video_enabled; - } - // Always reply with the new (now "current") state. - // This allows to also use this action to just get the unaltered current state - // by using a fromWidget request with: `ev.detail.data = {}` - widget!.api.transport.reply(ev.detail, newState); - }, - [audio, video], - ); - useEffect(() => { - // We setup a event listener for the widget action ElementWidgetActions.DeviceMute. - if (widget) { - // only setup the listener in widget mode - - widget.lazyActions.on( - ElementWidgetActions.DeviceMute, - onMuteStateChangeRequest, - ); - - return (): void => { - // return a call to `off` so that we always clean up our listener. - widget?.lazyActions.off( - ElementWidgetActions.DeviceMute, - onMuteStateChangeRequest, - ); - }; - } - }, [onMuteStateChangeRequest]); - - return useMemo(() => ({ audio, video }), [audio, video]); -} - -function allowJoinUnmuted(skipLobby: boolean, isJoined: boolean): boolean { - return ( - (!skipLobby && !isJoined) || import.meta.env.VITE_PACKAGE === "embedded" - ); -} From b1d143720aa322d9f263c4e3736a0dd5ebc0d468 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 17:08:51 -0400 Subject: [PATCH 27/46] Add comments to Async --- src/state/Async.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/state/Async.ts b/src/state/Async.ts index 79de41409..61871f785 100644 --- a/src/state/Async.ts +++ b/src/state/Async.ts @@ -7,8 +7,12 @@ Please see LICENSE in the repository root for full details. import { catchError, from, map, type Observable, of, startWith } from "rxjs"; -// TODO where are all the comments? ::cry:: -// There used to be an unitialized state!, a state might not start in loading +/** + * Data that may need to be loaded asynchronously. + * + * This type is for when you need to represent the current state of an operation + * involving Promises as **immutable data**. See the async$ function below. + */ export type Async = | { state: "loading" } | { state: "error"; value: Error } @@ -23,6 +27,11 @@ export function ready(value: A): Async { return { state: "ready", value }; } +/** + * Turn a Promise into an Observable async value. The Observable will have the + * value "loading" while the Promise is pending, "ready" when the Promise + * resolves, and "error" when the Promise rejects. + */ export function async$(promise: Promise): Observable> { return from(promise).pipe( map(ready), @@ -33,6 +42,9 @@ export function async$(promise: Promise): Observable> { ); } +/** + * If the async value is ready, apply the given function to the inner value. + */ export function mapAsync( async: Async, project: (value: A) => B, From e88474452fddb49432d47dd13cbc0f6e3c8eab7e Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 17:33:17 -0400 Subject: [PATCH 28/46] Correct / document some missing bits in tests --- src/room/GroupCallView.test.tsx | 2 ++ src/room/InCallView.test.tsx | 1 + src/room/InCallView.tsx | 2 +- src/room/VideoPreview.test.tsx | 4 ++-- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx index 8c4a276ae..ea14f5cf4 100644 --- a/src/room/GroupCallView.test.tsx +++ b/src/room/GroupCallView.test.tsx @@ -150,6 +150,7 @@ function createGroupCallView( const muteState = { audio: { enabled: false }, video: { enabled: false }, + // TODO-MULTI-SFU: This cast isn't valid, it's likely the cause of some current test failures } as unknown as MuteStates; const { getByText } = render( @@ -166,6 +167,7 @@ function createGroupCallView( rtcSession={rtcSession as unknown as MatrixRTCSession} muteStates={muteState} widget={widget} + // TODO-MULTI-SFU: Make joined and setJoined work joined={true} setJoined={function (value: boolean): void {}} /> diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index d26941202..131259da6 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -136,6 +136,7 @@ function createInCallView(): RenderResult & { const muteState = { audio: { enabled: false }, video: { enabled: false }, + // TODO-MULTI-SFU: This cast isn't valid, it's likely the cause of some current test failures } as unknown as MuteStates; const livekitRoom = mockLivekitRoom( { diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index dacb7eb15..658f9fbe9 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -205,7 +205,7 @@ export const InCallView: FC = ({ useReactionsSender(); useWakeLock(); - // TODO multi-sfu This is unused now?? + // TODO-MULTI-SFU This is unused now?? // const connectionState = useObservableEagerState(vm.livekitConnectionState$); // annoyingly we don't get the disconnection reason this way, diff --git a/src/room/VideoPreview.test.tsx b/src/room/VideoPreview.test.tsx index 17a05e344..dba657278 100644 --- a/src/room/VideoPreview.test.tsx +++ b/src/room/VideoPreview.test.tsx @@ -41,7 +41,7 @@ describe("VideoPreview", () => { const { queryByRole } = render( } />, @@ -53,7 +53,7 @@ describe("VideoPreview", () => { const { queryByRole } = render( } />, From 8778be83510a93d9c4e89dab93b25467020737f0 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 17:34:04 -0400 Subject: [PATCH 29/46] Fix doc comment typo --- src/state/PublishConnection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/state/PublishConnection.ts b/src/state/PublishConnection.ts index c35c71e46..9a2194833 100644 --- a/src/state/PublishConnection.ts +++ b/src/state/PublishConnection.ts @@ -36,7 +36,7 @@ import { Connection, type ConnectionOpts } from "./Connection.ts"; import { type ObservableScope } from "./ObservableScope.ts"; /** - * A connection to the publishing LiveKit.e. the local livekit room, the one the user is publishing to. + * A connection to the local LiveKit room, the one the user is publishing to. * This connection will publish the local user's audio and video tracks. */ export class PublishConnection extends Connection { From 3691e7120d572082af5560aa5539aa02a5eb6906 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 17:35:53 -0400 Subject: [PATCH 30/46] Restore a hidden 'null' state for the local transport/connection --- src/state/CallViewModel.ts | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 4d83d9972..815cbf178 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -562,16 +562,22 @@ export class CallViewModel extends ViewModel { /** * The transport over which we should be actively publishing our media. + * null when not joined. */ - private readonly localTransport$: Behavior> = + private readonly localTransport$: Behavior | null> = this.scope.behavior( this.transports$.pipe( map((transports) => transports?.local ?? null), - distinctUntilChanged(deepCompare), + distinctUntilChanged | null>(deepCompare), ), ); - private readonly localConnection$: Behavior> = + /** + * The local connection over which we will publish our media. It could + * possibly also have some remote users' media available on it. + * null when not joined. + */ + private readonly localConnection$: Behavior | null> = this.scope.behavior( this.localTransport$.pipe( map( @@ -807,15 +813,14 @@ export class CallViewModel extends ViewModel { // TODO: Move this logic into Connection/PublishConnection if possible this.localConnection$ .pipe( - switchMap((values) => { - if (values?.state !== "ready") return []; - const localConnection = values.value; + switchMap((localConnection) => { + if (localConnection?.state !== "ready") return []; const memberError = (): never => { throw new Error("No room member for call membership"); }; const localParticipant = { id: "local", - participant: localConnection.livekitRoom.localParticipant, + participant: localConnection.value.livekitRoom.localParticipant, member: this.matrixRoom.getMember(this.userId ?? "") ?? memberError(), }; @@ -823,7 +828,7 @@ export class CallViewModel extends ViewModel { return this.remoteConnections$.pipe( switchMap((remoteConnections) => combineLatest( - [localConnection, ...remoteConnections].map((c) => + [localConnection.value, ...remoteConnections].map((c) => c.publishingParticipants$.pipe( map((ps) => { const participants: { @@ -842,7 +847,7 @@ export class CallViewModel extends ViewModel { this.matrixRoom, )?.member ?? memberError(), })); - if (c === localConnection) + if (c === localConnection.value) participants.push(localParticipant); return { @@ -1974,7 +1979,10 @@ export class CallViewModel extends ViewModel { ); c.stop().catch((err) => { // TODO: better error handling - logger.error("MuteState: handler error", err); + logger.error( + `Fail to stop connection to ${c.localTransport.livekit_service_url}`, + err, + ); }); } for (const c of start) { From dee06a4b701ba8ca3b46ab70f575d93a85b3fea9 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 17:37:54 -0400 Subject: [PATCH 31/46] Remove unused useIsEarpiece hook --- src/MediaDevicesContext.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/MediaDevicesContext.ts b/src/MediaDevicesContext.ts index 3cf54c2ae..801219b01 100644 --- a/src/MediaDevicesContext.ts +++ b/src/MediaDevicesContext.ts @@ -23,14 +23,6 @@ export function useMediaDevices(): MediaDevices { return mediaDevices; } -export const useIsEarpiece = (): boolean => { - const devices = useMediaDevices(); - const audioOutput = useObservableEagerState(devices.audioOutput.selected$); - const available = useObservableEagerState(devices.audioOutput.available$); - if (!audioOutput?.id) return false; - return available.get(audioOutput.id)?.type === "earpiece"; -}; - /** * A convenience hook to get the audio node configuration for the earpiece. * It will check the `useAsEarpiece` of the `audioOutput` device and return From dcc3ab641f59ae986edca267d9b7787ed9c546fa Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 17:40:03 -0400 Subject: [PATCH 32/46] Remove MockedObject from mockMediaDevices type signature --- src/utils/test.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/utils/test.ts b/src/utils/test.ts index d0e08dd89..6e0e95c97 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { map, type Observable, of, type SchedulerLike } from "rxjs"; import { type RunHelpers, TestScheduler } from "rxjs/testing"; -import { expect, type MockedObject, vi, vitest } from "vitest"; +import { expect, vi, vitest } from "vitest"; import { type RoomMember, type Room as MatrixRoom, @@ -418,15 +418,13 @@ export const deviceStub = { select(): void {}, }; -export function mockMediaDevices( - data: Partial, -): MockedObject { - return vi.mocked({ +export function mockMediaDevices(data: Partial): MediaDevices { + return { audioInput: deviceStub, audioOutput: deviceStub, videoInput: deviceStub, ...data, - } as MediaDevices); + } as MediaDevices; } export function mockMuteStates( From 00daf834b655140d402515beb52505946e79b7d0 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 17:53:11 -0400 Subject: [PATCH 33/46] Remove local participant case (now enforced by types) from audio tests --- src/livekit/MatrixAudioRenderer.test.tsx | 44 +++++++----------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index 9519ccc25..b78b274d0 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -65,27 +65,25 @@ let tracks: TrackReference[] = []; function renderTestComponent( rtcMembers: { userId: string; deviceId: string }[], - livekitParticipantIdentities: ({ id: string; isLocal?: boolean } | string)[], + livekitParticipantIdentities: string[], ): RenderResult { - const liveKitParticipants = livekitParticipantIdentities.map((p) => { - const identity = typeof p === "string" ? p : p.id; - const isLocal = typeof p === "string" ? false : (p.isLocal ?? false); - return vi.mocked({ - identity, - isLocal, - } as unknown as RemoteParticipant); - }); + const liveKitParticipants = livekitParticipantIdentities.map( + (identity) => + ({ + identity, + }) as unknown as RemoteParticipant, + ); const participants = rtcMembers.flatMap(({ userId, deviceId }) => { const p = liveKitParticipants.find( (p) => p.identity === `${userId}:${deviceId}`, ); return p === undefined ? [] : [p]; }); - const livekitRoom = vi.mocked({ + const livekitRoom = { remoteParticipants: new Map( liveKitParticipants.map((p) => [p.identity, p]), ), - } as unknown as Room); + } as unknown as Room; tracks = participants.map((p) => mockTrack(p)); @@ -121,7 +119,7 @@ it("should not render without member", () => { const TEST_CASES: { rtcUsers: { userId: string; deviceId: string }[]; - livekitParticipantIdentities: (string | { id: string; isLocal?: boolean })[]; + livekitParticipantIdentities: string[]; expectedAudioTracks: number; }[] = [ { @@ -130,27 +128,9 @@ const TEST_CASES: { { userId: "@alice", deviceId: "DEV1" }, { userId: "@bob", deviceId: "DEV0" }, ], - livekitParticipantIdentities: [ - { id: "@alice:DEV0" }, - "@bob:DEV0", - "@alice:DEV1", - ], + livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@alice:DEV1"], expectedAudioTracks: 3, }, - // Alice DEV0 is local participant, should not render - { - rtcUsers: [ - { userId: "@alice", deviceId: "DEV0" }, - { userId: "@alice", deviceId: "DEV1" }, - { userId: "@bob", deviceId: "DEV0" }, - ], - livekitParticipantIdentities: [ - { id: "@alice:DEV0", isLocal: true }, - "@bob:DEV0", - "@alice:DEV1", - ], - expectedAudioTracks: 2, - }, // Charlie is a rtc member but not in livekit { rtcUsers: [ @@ -158,7 +138,7 @@ const TEST_CASES: { { userId: "@bob", deviceId: "DEV0" }, { userId: "@charlie", deviceId: "DEV0" }, ], - livekitParticipantIdentities: ["@alice:DEV0", { id: "@bob:DEV0" }], + livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0"], expectedAudioTracks: 2, }, // Charlie is in livekit but not rtc member From 5be3b9150959b3fa15396014ca7493f258101d3f Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 18:10:26 -0400 Subject: [PATCH 34/46] Fix focus connection state typo, simplify its initialization --- src/state/CallViewModel.ts | 2 +- src/state/Connection.test.ts | 28 ++++++---------------------- src/state/Connection.ts | 25 ++++++++++--------------- 3 files changed, 17 insertions(+), 38 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 815cbf178..4cb975195 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -607,7 +607,7 @@ export class CallViewModel extends ViewModel { switchMap((c) => c?.state === "ready" ? // TODO mapping to ConnectionState for compatibility, but we should use the full state? - c.value.focusedConnectionState$.pipe( + c.value.focusConnectionState$.pipe( map((s) => { if (s.state === "ConnectedToLkRoom") return s.connectionState; return ConnectionState.Disconnected; diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index 699422707..ecafb5eef 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -159,7 +159,7 @@ describe("Start connection states", () => { }; const connection = new RemoteConnection(opts, undefined); - expect(connection.focusedConnectionState$.getValue().state).toEqual( + expect(connection.focusConnectionState$.getValue().state).toEqual( "Initialized", ); }); @@ -179,7 +179,7 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; - connection.focusedConnectionState$.subscribe((value) => { + connection.focusConnectionState$.subscribe((value) => { capturedStates.push(value); }); @@ -231,7 +231,7 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; - connection.focusedConnectionState$.subscribe((value) => { + connection.focusConnectionState$.subscribe((value) => { capturedStates.push(value); }); @@ -287,7 +287,7 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; - connection.focusedConnectionState$.subscribe((value) => { + connection.focusConnectionState$.subscribe((value) => { capturedStates.push(value); }); @@ -343,7 +343,7 @@ describe("Start connection states", () => { const connection = setupRemoteConnection(); const capturedState: FocusConnectionState[] = []; - connection.focusedConnectionState$.subscribe((value) => { + connection.focusConnectionState$.subscribe((value) => { capturedState.push(value); }); @@ -368,7 +368,7 @@ describe("Start connection states", () => { await connection.start(); let capturedState: FocusConnectionState[] = []; - connection.focusedConnectionState$.subscribe((value) => { + connection.focusConnectionState$.subscribe((value) => { capturedState.push(value); }); @@ -417,12 +417,6 @@ describe("Start connection states", () => { vi.useFakeTimers(); const connection = setupRemoteConnection(); - - let capturedState: FocusConnectionState[] = []; - connection.focusedConnectionState$.subscribe((value) => { - capturedState.push(value); - }); - await connection.start(); const stopSpy = vi.spyOn(connection, "stop"); @@ -430,16 +424,6 @@ describe("Start connection states", () => { expect(stopSpy).toHaveBeenCalled(); expect(fakeLivekitRoom.disconnect).toHaveBeenCalled(); - - /// Ensures that focusedConnectionState$ is bound to the scope. - capturedState = []; - // the subscription should be closed, and no new state should be received - // @ts-expect-error: Accessing private field for testing purposes - connection._focusedConnectionState$.next({ state: "Initialized" }); - // @ts-expect-error: Accessing private field for testing purposes - connection._focusedConnectionState$.next({ state: "ConnectingToLkRoom" }); - - expect(capturedState.length).toEqual(0); }); }); diff --git a/src/state/Connection.ts b/src/state/Connection.ts index e5e108b7d..ac381d560 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -66,13 +66,14 @@ export type FocusConnectionState = */ export class Connection { // Private Behavior - private readonly _focusedConnectionState$ = + private readonly _focusConnectionState$ = new BehaviorSubject({ state: "Initialized" }); /** * The current state of the connection to the focus server. */ - public readonly focusedConnectionState$: Behavior; + public readonly focusConnectionState$: Behavior = + this._focusConnectionState$; /** * Whether the connection has been stopped. @@ -91,7 +92,7 @@ export class Connection { public async start(): Promise { this.stopped = false; try { - this._focusedConnectionState$.next({ + this._focusConnectionState$.next({ state: "FetchingConfig", focus: this.localTransport, }); @@ -100,7 +101,7 @@ export class Connection { // If we were stopped while fetching the config, don't proceed to connect if (this.stopped) return; - this._focusedConnectionState$.next({ + this._focusConnectionState$.next({ state: "ConnectingToLkRoom", focus: this.localTransport, }); @@ -108,13 +109,13 @@ export class Connection { // If we were stopped while connecting, don't proceed to update state. if (this.stopped) return; - this._focusedConnectionState$.next({ + this._focusConnectionState$.next({ state: "ConnectedToLkRoom", focus: this.localTransport, connectionState: this.livekitRoom.state, }); } catch (error) { - this._focusedConnectionState$.next({ + this._focusConnectionState$.next({ state: "FailedToStart", error: error instanceof Error ? error : new Error(`${error}`), focus: this.localTransport, @@ -139,7 +140,7 @@ export class Connection { public async stop(): Promise { if (this.stopped) return; await this.livekitRoom.disconnect(); - this._focusedConnectionState$.next({ + this._focusConnectionState$.next({ state: "Stopped", focus: this.localTransport, }); @@ -172,15 +173,9 @@ export class Connection { ) { const { transport, client, scope, remoteTransports$ } = opts; - this.livekitRoom = livekitRoom; this.localTransport = transport; this.client = client; - this.focusedConnectionState$ = scope.behavior( - this._focusedConnectionState$, - { state: "Initialized" }, - ); - const participantsIncludingSubscribers$ = scope.behavior( connectedParticipantsObserver(this.livekitRoom), [], @@ -212,10 +207,10 @@ export class Connection { scope .behavior(connectionStateObserver(this.livekitRoom)) .subscribe((connectionState) => { - const current = this._focusedConnectionState$.value; + const current = this._focusConnectionState$.value; // Only update the state if we are already connected to the LiveKit room. if (current.state === "ConnectedToLkRoom") { - this._focusedConnectionState$.next({ + this._focusConnectionState$.next({ state: "ConnectedToLkRoom", connectionState, focus: current.focus, From 64c2e5911c37f3ff73d8966756e721768df0567e Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 18:17:42 -0400 Subject: [PATCH 35/46] Update outdated comment --- src/state/Connection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/state/Connection.ts b/src/state/Connection.ts index ac381d560..b7864677f 100644 --- a/src/state/Connection.ts +++ b/src/state/Connection.ts @@ -149,7 +149,7 @@ export class Connection { /** * An observable of the participants that are publishing on this connection. - * This is derived from `participantsIncludingSubscribers$` and `membershipsFocusMap$`. + * This is derived from `participantsIncludingSubscribers$` and `remoteTransports$`. * It filters the participants to only those that are associated with a membership that claims to publish on this connection. */ public readonly publishingParticipants$; From 2c576a7477d509a293e9bf59b48359934816cf26 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 18:58:03 -0400 Subject: [PATCH 36/46] Clean up subscriptions in Connection tests --- src/state/Connection.test.ts | 44 +++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/state/Connection.test.ts b/src/state/Connection.test.ts index ecafb5eef..14c422060 100644 --- a/src/state/Connection.test.ts +++ b/src/state/Connection.test.ts @@ -12,6 +12,7 @@ import { it, type Mock, type MockedObject, + onTestFinished, vi, } from "vitest"; import { BehaviorSubject, of } from "rxjs"; @@ -179,9 +180,10 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; - connection.focusConnectionState$.subscribe((value) => { + const s = connection.focusConnectionState$.subscribe((value) => { capturedStates.push(value); }); + onTestFinished(() => s.unsubscribe()); const deferred = Promise.withResolvers(); @@ -231,9 +233,10 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; - connection.focusConnectionState$.subscribe((value) => { + const s = connection.focusConnectionState$.subscribe((value) => { capturedStates.push(value); }); + onTestFinished(() => s.unsubscribe()); const deferredSFU = Promise.withResolvers(); // mock the /sfu/get call @@ -287,9 +290,10 @@ describe("Start connection states", () => { const connection = new RemoteConnection(opts, undefined); const capturedStates: FocusConnectionState[] = []; - connection.focusConnectionState$.subscribe((value) => { + const s = connection.focusConnectionState$.subscribe((value) => { capturedStates.push(value); }); + onTestFinished(() => s.unsubscribe()); const deferredSFU = Promise.withResolvers(); // mock the /sfu/get call @@ -342,21 +346,22 @@ describe("Start connection states", () => { const connection = setupRemoteConnection(); - const capturedState: FocusConnectionState[] = []; - connection.focusConnectionState$.subscribe((value) => { - capturedState.push(value); + const capturedStates: FocusConnectionState[] = []; + const s = connection.focusConnectionState$.subscribe((value) => { + capturedStates.push(value); }); + onTestFinished(() => s.unsubscribe()); await connection.start(); await vi.runAllTimersAsync(); - const initialState = capturedState.shift(); + const initialState = capturedStates.shift(); expect(initialState?.state).toEqual("Initialized"); - const fetchingState = capturedState.shift(); + const fetchingState = capturedStates.shift(); expect(fetchingState?.state).toEqual("FetchingConfig"); - const connectingState = capturedState.shift(); + const connectingState = capturedStates.shift(); expect(connectingState?.state).toEqual("ConnectingToLkRoom"); - const connectedState = capturedState.shift(); + const connectedState = capturedStates.shift(); expect(connectedState?.state).toEqual("ConnectedToLkRoom"); }); @@ -367,10 +372,11 @@ describe("Start connection states", () => { await connection.start(); - let capturedState: FocusConnectionState[] = []; - connection.focusConnectionState$.subscribe((value) => { - capturedState.push(value); + let capturedStates: FocusConnectionState[] = []; + const s = connection.focusConnectionState$.subscribe((value) => { + capturedStates.push(value); }); + onTestFinished(() => s.unsubscribe()); const states = [ ConnectionState.Disconnected, @@ -386,7 +392,7 @@ describe("Start connection states", () => { } for (const state of states) { - const s = capturedState.shift(); + const s = capturedStates.shift(); expect(s?.state).toEqual("ConnectedToLkRoom"); const connectedState = s as FocusConnectionState & { state: "ConnectedToLkRoom"; @@ -404,12 +410,12 @@ describe("Start connection states", () => { // If the state is not ConnectedToLkRoom, no events should be relayed anymore await connection.stop(); - capturedState = []; + capturedStates = []; for (const state of states) { fakeRoomEventEmiter.emit(RoomEvent.ConnectionStateChanged, state); } - expect(capturedState.length).toEqual(0); + expect(capturedStates.length).toEqual(0); }); it("shutting down the scope should stop the connection", async () => { @@ -452,7 +458,7 @@ describe("Publishing participants observations", () => { participant: RemoteParticipant; membership: CallMembership; }[][] = []; - connection.publishingParticipants$.subscribe((publishers) => { + const s = connection.publishingParticipants$.subscribe((publishers) => { observedPublishers.push(publishers); if ( publishers.some( @@ -469,6 +475,7 @@ describe("Publishing participants observations", () => { danIsAPublisher.resolve(); } }); + onTestFinished(() => s.unsubscribe()); // The publishingParticipants$ observable is derived from the current members of the // livekitRoom and the rtc membership in order to publish the members that are publishing // on this connection. @@ -578,9 +585,10 @@ describe("Publishing participants observations", () => { participant: RemoteParticipant; membership: CallMembership; }[][] = []; - connection.publishingParticipants$.subscribe((publishers) => { + const s = connection.publishingParticipants$.subscribe((publishers) => { observedPublishers.push(publishers); }); + onTestFinished(() => s.unsubscribe()); let participants: RemoteParticipant[] = [ fakeRemoteLivekitParticipant("@bob:example.org:DEV111"), From 85ffe68d98b887b4e6f60601977e84f223e04d1a Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 Oct 2025 19:20:21 -0400 Subject: [PATCH 37/46] Remove outdated comment --- src/state/MuteStates.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/state/MuteStates.ts b/src/state/MuteStates.ts index 8a0258825..50be5e056 100644 --- a/src/state/MuteStates.ts +++ b/src/state/MuteStates.ts @@ -138,7 +138,6 @@ class MuteState { ) {} } -// TODO there is another MuteStates in src/room/MuteStates.tsx ?? why export class MuteStates { public readonly audio = new MuteState( this.scope, From 4c6b960da34e98df5f43cf5e028fc98755fe31c6 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 9 Oct 2025 11:00:45 +0200 Subject: [PATCH 38/46] fix: use correct TestEachFunction --- src/livekit/MatrixAudioRenderer.test.tsx | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index b78b274d0..83b3f73ac 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -152,15 +152,14 @@ const TEST_CASES: { }, ]; -TEST_CASES.forEach( - ({ rtcUsers, livekitParticipantIdentities, expectedAudioTracks }, index) => { - it(`should render sound test cases #${index + 1}`, () => { - const { queryAllByTestId } = renderTestComponent( - rtcUsers, - livekitParticipantIdentities, - ); - expect(queryAllByTestId("audio")).toHaveLength(expectedAudioTracks); - }); +it.each(TEST_CASES)( + `should render sound test cases %s`, + ({ rtcUsers, livekitParticipantIdentities, expectedAudioTracks }) => { + const { queryAllByTestId } = renderTestComponent( + rtcUsers, + livekitParticipantIdentities, + ); + expect(queryAllByTestId("audio")).toHaveLength(expectedAudioTracks); }, ); From 7cbb1ec1e8069453d987c215ec7a9cd9a2045064 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 9 Oct 2025 15:33:25 +0200 Subject: [PATCH 39/46] Simplify AudioRenderer and add more tests --- src/livekit/MatrixAudioRenderer.test.tsx | 110 ++++++++++++++++++++--- src/livekit/MatrixAudioRenderer.tsx | 83 ++++++----------- src/room/InCallView.tsx | 2 +- src/utils/test.ts | 12 ++- 4 files changed, 133 insertions(+), 74 deletions(-) diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index 83b3f73ac..049add97d 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -14,8 +14,8 @@ import { import { type Participant, type RemoteAudioTrack, - type RemoteParticipant, type Room, + Track, } from "livekit-client"; import { type ReactNode } from "react"; import { useTracks } from "@livekit/components-react"; @@ -23,7 +23,11 @@ import { useTracks } from "@livekit/components-react"; import { testAudioContext } from "../useAudioContext.test"; import * as MediaDevicesContext from "../MediaDevicesContext"; import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer"; -import { mockMediaDevices, mockTrack } from "../utils/test"; +import { + mockMediaDevices, + mockRemoteParticipant, + mockTrack, +} from "../utils/test"; export const TestAudioContextConstructor = vi.fn(() => testAudioContext); @@ -61,17 +65,20 @@ let tracks: TrackReference[] = []; * * @param rtcMembers - Array of active rtc members with userId and deviceId. * @param livekitParticipantIdentities - Array of livekit participant (that are publishing). + * @param explicitTracks - Array of tracks available in livekit, if not provided, one audio track per livekitParticipantIdentities will be created. * */ function renderTestComponent( rtcMembers: { userId: string; deviceId: string }[], livekitParticipantIdentities: string[], + explicitTracks?: { + participantId: string; + kind: Track.Kind; + source: Track.Source; + }[], ): RenderResult { - const liveKitParticipants = livekitParticipantIdentities.map( - (identity) => - ({ - identity, - }) as unknown as RemoteParticipant, + const liveKitParticipants = livekitParticipantIdentities.map((identity) => + mockRemoteParticipant({ identity }), ); const participants = rtcMembers.flatMap(({ userId, deviceId }) => { const p = liveKitParticipants.find( @@ -85,13 +92,22 @@ function renderTestComponent( ), } as unknown as Room; - tracks = participants.map((p) => mockTrack(p)); + if (explicitTracks?.length ?? 0 > 0) { + tracks = explicitTracks!.map(({ participantId, source, kind }) => { + const participant = + liveKitParticipants.find((p) => p.identity === participantId) ?? + mockRemoteParticipant({ identity: participantId }); + return mockTrack(participant, kind, source); + }); + } else { + tracks = participants.map((p) => mockTrack(p)); + } vi.mocked(useTracks).mockReturnValue(tracks); return render( p.identity)} livekitRoom={livekitRoom} url={""} /> @@ -118,11 +134,18 @@ it("should not render without member", () => { }); const TEST_CASES: { + name: string; rtcUsers: { userId: string; deviceId: string }[]; livekitParticipantIdentities: string[]; + explicitTracks?: { + participantId: string; + kind: Track.Kind; + source: Track.Source; + }[]; expectedAudioTracks: number; }[] = [ { + name: "single user single device", rtcUsers: [ { userId: "@alice", deviceId: "DEV0" }, { userId: "@alice", deviceId: "DEV1" }, @@ -133,6 +156,7 @@ const TEST_CASES: { }, // Charlie is a rtc member but not in livekit { + name: "Charlie is rtc member but not in livekit", rtcUsers: [ { userId: "@alice", deviceId: "DEV0" }, { userId: "@bob", deviceId: "DEV0" }, @@ -143,6 +167,7 @@ const TEST_CASES: { }, // Charlie is in livekit but not rtc member { + name: "Charlie is in livekit but not rtc member", rtcUsers: [ { userId: "@alice", deviceId: "DEV0" }, { userId: "@bob", deviceId: "DEV0" }, @@ -150,14 +175,77 @@ const TEST_CASES: { livekitParticipantIdentities: ["@alice:DEV0", "@bob:DEV0", "@charlie:DEV0"], expectedAudioTracks: 2, }, + { + name: "no audio track, only video track", + rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }], + livekitParticipantIdentities: ["@alice:DEV0"], + explicitTracks: [ + { + participantId: "@alice:DEV0", + kind: Track.Kind.Video, + source: Track.Source.Camera, + }, + ], + expectedAudioTracks: 0, + }, + { + name: "Audio track from unknown source", + rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }], + livekitParticipantIdentities: ["@alice:DEV0"], + explicitTracks: [ + { + participantId: "@alice:DEV0", + kind: Track.Kind.Audio, + source: Track.Source.Unknown, + }, + ], + expectedAudioTracks: 1, + }, + { + name: "Audio track from other device", + rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }], + livekitParticipantIdentities: ["@alice:DEV0"], + explicitTracks: [ + { + participantId: "@alice:DEV1", + kind: Track.Kind.Audio, + source: Track.Source.Microphone, + }, + ], + expectedAudioTracks: 0, + }, + { + name: "two audio tracks, microphone and screenshare", + rtcUsers: [{ userId: "@alice", deviceId: "DEV0" }], + livekitParticipantIdentities: ["@alice:DEV0"], + explicitTracks: [ + { + participantId: "@alice:DEV0", + kind: Track.Kind.Audio, + source: Track.Source.Microphone, + }, + { + participantId: "@alice:DEV0", + kind: Track.Kind.Audio, + source: Track.Source.ScreenShareAudio, + }, + ], + expectedAudioTracks: 2, + }, ]; it.each(TEST_CASES)( - `should render sound test cases %s`, - ({ rtcUsers, livekitParticipantIdentities, expectedAudioTracks }) => { + `should render sound test cases $name`, + ({ + rtcUsers, + livekitParticipantIdentities, + explicitTracks, + expectedAudioTracks, + }) => { const { queryAllByTestId } = renderTestComponent( rtcUsers, livekitParticipantIdentities, + explicitTracks, ); expect(queryAllByTestId("audio")).toHaveLength(expectedAudioTracks); }, diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index fb1400b43..5b1149e99 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -6,23 +6,20 @@ Please see LICENSE in the repository root for full details. */ import { getTrackReferenceId } from "@livekit/components-core"; -import { - type RemoteParticipant, - type Room as LivekitRoom, -} from "livekit-client"; +import { type Room as LivekitRoom } from "livekit-client"; import { type RemoteAudioTrack, Track } from "livekit-client"; -import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; +import { useEffect, useMemo, useState, type ReactNode } from "react"; import { useTracks, AudioTrack, type AudioTrackProps, } from "@livekit/components-react"; import { logger } from "matrix-js-sdk/lib/logger"; +import { type ParticipantId } from "matrix-js-sdk/lib/matrixrtc"; import { useEarpieceAudioConfig } from "../MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; import * as controls from "../controls"; -import {} from "@livekit/components-core"; export interface MatrixAudioRendererProps { /** @@ -31,11 +28,11 @@ export interface MatrixAudioRendererProps { url: string; livekitRoom: LivekitRoom; /** - * The list of participants to render audio for. + * The list of participant identities to render audio for. * This list needs to be composed based on the matrixRTC members so that we do not play audio from users - * that are not expected to be in the rtc session. + * that are not expected to be in the rtc session (local user is excluded). */ - participants: RemoteParticipant[]; + validIdentities: ParticipantId[]; /** * If set to `true`, mutes all audio tracks rendered by the component. * @remarks @@ -44,6 +41,7 @@ export interface MatrixAudioRendererProps { muted?: boolean; } +const prefixedLogger = logger.getChild("[MatrixAudioRenderer]"); /** * Takes care of handling remote participants’ audio tracks and makes sure that microphones and screen share are audible. * @@ -60,35 +58,9 @@ export interface MatrixAudioRendererProps { export function LivekitRoomAudioRenderer({ url, livekitRoom, - participants, + validIdentities, muted, }: MatrixAudioRendererProps): ReactNode { - // This is the list of valid identities that are allowed to play audio. - // It is derived from the list of matrix rtc members. - const validIdentities = useMemo( - () => new Set(participants.map((p) => p.identity)), - [participants], - ); - - const loggedInvalidIdentities = useRef(new Set()); - - /** - * Log an invalid livekit track identity. - * A invalid identity is one that does not match any of the matrix rtc members. - * - * @param identity The identity of the track that is invalid - * @param validIdentities The list of valid identities - */ - const logInvalid = (identity: string): void => { - if (loggedInvalidIdentities.current.has(identity)) return; - logger.warn( - `[MatrixAudioRenderer] Audio track ${identity} from ${url} has no matching matrix call member`, - `current members: ${participants.map((p) => p.identity)}`, - `track will not get rendered`, - ); - loggedInvalidIdentities.current.add(identity); - }; - const tracks = useTracks( [ Track.Source.Microphone, @@ -100,28 +72,23 @@ export function LivekitRoomAudioRenderer({ onlySubscribed: true, room: livekitRoom, }, - ).filter((ref) => { - const isValid = validIdentities.has(ref.participant.identity); - if (!isValid && !ref.participant.isLocal) - logInvalid(ref.participant.identity); - return ( - !ref.participant.isLocal && - ref.publication.kind === Track.Kind.Audio && - isValid - ); - }); - - useEffect(() => { - if ( - loggedInvalidIdentities.current.size && - tracks.every((t) => validIdentities.has(t.participant.identity)) - ) { - logger.debug( - `[MatrixAudioRenderer] All audio tracks from ${url} have a matching matrix call member identity.`, - ); - loggedInvalidIdentities.current.clear(); - } - }, [tracks, validIdentities, url]); + ) + // Only keep audio tracks + .filter((ref) => ref.publication.kind === Track.Kind.Audio) + // Only keep tracks from participants that are in the validIdentities list + .filter((ref) => { + const isValid = validIdentities.includes(ref.participant.identity); + if (!isValid) { + // Log that there is an invalid identity, that means that someone is publishing audio that is not expected to be in the call. + prefixedLogger.warn( + `Audio track ${ref.participant.identity} from ${url} has no matching matrix call member`, + `current members: ${validIdentities.join()}`, + `track will not get rendered`, + ); + return false; + } + return true; + }); // This component is also (in addition to the "only play audio for connected members" logic above) // responsible for mimicking earpiece audio on iPhones. diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 658f9fbe9..fd631baec 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -865,7 +865,7 @@ export const InCallView: FC = ({ key={url} url={url} livekitRoom={livekitRoom} - participants={participants} + validIdentities={participants.map((p) => p.identity)} muted={muteAllAudio} /> ))} diff --git a/src/utils/test.ts b/src/utils/test.ts index 6e0e95c97..508559c2b 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -393,13 +393,17 @@ export class MockRTCSession extends TypedEventEmitter< } } -export const mockTrack = (participant: Participant): TrackReference => +export const mockTrack = ( + participant: Participant, + kind?: Track.Kind, + source?: Track.Source, +): TrackReference => ({ participant, publication: { - kind: Track.Kind.Audio, - source: "mic", - trackSid: "123", + kind: kind ?? Track.Kind.Audio, + source: source ?? Track.Source.Microphone, + trackSid: `123##${participant.identity}`, track: { attach: vi.fn(), detach: vi.fn(), From a500915c436415ed91b9130820da492bab3d9876 Mon Sep 17 00:00:00 2001 From: Valere Date: Thu, 9 Oct 2025 19:24:44 +0200 Subject: [PATCH 40/46] test: Fix mute test, behavior change from setMuted to setAudioEnabled useCallViewKeyboardShortcuts() changed a param from `setMicrophoneMuted` to `setAudioEnabled`, the boolean arg of the callback is inverse tht it used to be --- src/useCallViewKeyboardShortcuts.test.tsx | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/useCallViewKeyboardShortcuts.test.tsx b/src/useCallViewKeyboardShortcuts.test.tsx index 86e1b03f5..e22380d1f 100644 --- a/src/useCallViewKeyboardShortcuts.test.tsx +++ b/src/useCallViewKeyboardShortcuts.test.tsx @@ -23,14 +23,14 @@ import { // The TestComponent just wraps a button around that hook. interface TestComponentProps { - setMicrophoneMuted?: (muted: boolean) => void; + setAudioEnabled?: (enabled: boolean) => void; onButtonClick?: () => void; sendReaction?: () => void; toggleHandRaised?: () => void; } const TestComponent: FC = ({ - setMicrophoneMuted = (): void => {}, + setAudioEnabled = (): void => {}, onButtonClick = (): void => {}, sendReaction = (reaction: ReactionOption): void => {}, toggleHandRaised = (): void => {}, @@ -40,7 +40,7 @@ const TestComponent: FC = ({ ref, () => {}, () => {}, - setMicrophoneMuted, + setAudioEnabled, sendReaction, toggleHandRaised, ); @@ -57,12 +57,13 @@ test("spacebar unmutes", async () => { render( (muted = false)} - setMicrophoneMuted={(m) => { - muted = m; + setAudioEnabled={(m) => { + muted = !m; }} />, ); + expect(muted).toBe(true); await user.keyboard("[Space>]"); expect(muted).toBe(false); await user.keyboard("[/Space]"); @@ -73,15 +74,15 @@ test("spacebar unmutes", async () => { test("spacebar prioritizes pressing a button", async () => { const user = userEvent.setup(); - const setMuted = vi.fn(); + const setAudioEnabled = vi.fn(); const onClick = vi.fn(); render( - , + , ); await user.tab(); // Focus the button await user.keyboard("[Space]"); - expect(setMuted).not.toBeCalled(); + expect(setAudioEnabled).not.toBeCalled(); expect(onClick).toBeCalled(); }); @@ -129,7 +130,7 @@ test("unmuting happens in place of the default action", async () => { tabIndex={0} onKeyDown={(e) => defaultPrevented(e.isDefaultPrevented())} > - {}} /> + {}} /> , ); From 6710f4c72ae54b5ba570b057e0314b618a32a688 Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 10 Oct 2025 11:09:41 +0200 Subject: [PATCH 41/46] Test: Fix mocking to fix failing tests --- src/button/ReactionToggleButton.test.tsx | 3 +-- src/reactions/ReactionsReader.test.tsx | 21 +++++++-------------- src/room/GroupCallView.test.tsx | 6 +++--- src/room/InCallView.test.tsx | 5 ++--- src/state/CallViewModel.test.ts | 3 +-- src/state/CallViewModel.ts | 14 ++++++++++---- src/utils/test-viewmodel.ts | 11 ++++------- src/utils/test.ts | 24 ++++++++++++++++++++++-- 8 files changed, 50 insertions(+), 37 deletions(-) diff --git a/src/button/ReactionToggleButton.test.tsx b/src/button/ReactionToggleButton.test.tsx index 269eabedf..b1af7ec8b 100644 --- a/src/button/ReactionToggleButton.test.tsx +++ b/src/button/ReactionToggleButton.test.tsx @@ -10,7 +10,6 @@ import { expect, test } from "vitest"; import { TooltipProvider } from "@vector-im/compound-web"; import { userEvent } from "@testing-library/user-event"; import { type ReactNode } from "react"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import { ReactionToggleButton } from "./ReactionToggleButton"; import { ElementCallReactionEventType } from "../reactions"; @@ -33,7 +32,7 @@ function TestComponent({ diff --git a/src/reactions/ReactionsReader.test.tsx b/src/reactions/ReactionsReader.test.tsx index b8acf5c75..01815c820 100644 --- a/src/reactions/ReactionsReader.test.tsx +++ b/src/reactions/ReactionsReader.test.tsx @@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details. import { renderHook } from "@testing-library/react"; import { afterEach, test, vitest } from "vitest"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import { RoomEvent as MatrixRoomEvent, MatrixEvent, @@ -38,7 +37,7 @@ test("handles a hand raised reaction", () => { withTestScheduler(({ schedule, expectObservable }) => { renderHook(() => { const { raisedHands$ } = new ReactionsReader( - rtcSession as unknown as MatrixRTCSession, + rtcSession.asMockedSession(), ); schedule("ab", { a: () => {}, @@ -86,7 +85,7 @@ test("handles a redaction", () => { withTestScheduler(({ schedule, expectObservable }) => { renderHook(() => { const { raisedHands$ } = new ReactionsReader( - rtcSession as unknown as MatrixRTCSession, + rtcSession.asMockedSession(), ); schedule("abc", { a: () => {}, @@ -149,7 +148,7 @@ test("handles waiting for event decryption", () => { withTestScheduler(({ schedule, expectObservable }) => { renderHook(() => { const { raisedHands$ } = new ReactionsReader( - rtcSession as unknown as MatrixRTCSession, + rtcSession.asMockedSession(), ); schedule("abc", { a: () => {}, @@ -218,7 +217,7 @@ test("hands rejecting events without a proper membership", () => { withTestScheduler(({ schedule, expectObservable }) => { renderHook(() => { const { raisedHands$ } = new ReactionsReader( - rtcSession as unknown as MatrixRTCSession, + rtcSession.asMockedSession(), ); schedule("ab", { a: () => {}, @@ -262,9 +261,7 @@ test("handles a reaction", () => { withTestScheduler(({ schedule, time, expectObservable }) => { renderHook(() => { - const { reactions$ } = new ReactionsReader( - rtcSession as unknown as MatrixRTCSession, - ); + const { reactions$ } = new ReactionsReader(rtcSession.asMockedSession()); schedule(`abc`, { a: () => {}, b: () => { @@ -320,9 +317,7 @@ test("ignores bad reaction events", () => { withTestScheduler(({ schedule, expectObservable }) => { renderHook(() => { - const { reactions$ } = new ReactionsReader( - rtcSession as unknown as MatrixRTCSession, - ); + const { reactions$ } = new ReactionsReader(rtcSession.asMockedSession()); schedule("ab", { a: () => {}, b: () => { @@ -444,9 +439,7 @@ test("that reactions cannot be spammed", () => { withTestScheduler(({ schedule, expectObservable }) => { renderHook(() => { - const { reactions$ } = new ReactionsReader( - rtcSession as unknown as MatrixRTCSession, - ); + const { reactions$ } = new ReactionsReader(rtcSession.asMockedSession()); schedule("abcd", { a: () => {}, b: () => { diff --git a/src/room/GroupCallView.test.tsx b/src/room/GroupCallView.test.tsx index ea14f5cf4..37f5c8503 100644 --- a/src/room/GroupCallView.test.tsx +++ b/src/room/GroupCallView.test.tsx @@ -117,7 +117,7 @@ function createGroupCallView( widget: WidgetHelpers | null, joined = true, ): { - rtcSession: MockRTCSession; + rtcSession: MatrixRTCSession; getByText: ReturnType["getByText"]; } { const client = { @@ -164,7 +164,7 @@ function createGroupCallView( preload={false} skipLobby={false} header={HeaderStyle.Standard} - rtcSession={rtcSession as unknown as MatrixRTCSession} + rtcSession={rtcSession.asMockedSession()} muteStates={muteState} widget={widget} // TODO-MULTI-SFU: Make joined and setJoined work @@ -178,7 +178,7 @@ function createGroupCallView( ); return { getByText, - rtcSession, + rtcSession: rtcSession.asMockedSession(), }; } diff --git a/src/room/InCallView.test.tsx b/src/room/InCallView.test.tsx index 131259da6..6b897c0d4 100644 --- a/src/room/InCallView.test.tsx +++ b/src/room/InCallView.test.tsx @@ -15,7 +15,6 @@ import { } from "vitest"; import { act, render, type RenderResult } from "@testing-library/react"; import { type MatrixClient, JoinRule, type RoomState } from "matrix-js-sdk"; -import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container"; import { type LocalParticipant } from "livekit-client"; import { of } from "rxjs"; @@ -154,14 +153,14 @@ function createInCallView(): RenderResult & { >({}); const vm = new CallViewModel( - rtcSession as unknown as MatrixRTCSession, + rtcSession.asMockedSession(), room, mediaDevices, muteStates, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 4cb975195..c3cf7ff36 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -541,7 +541,9 @@ export class CallViewModel extends ViewModel { const oldest = this.matrixRTCSession.getOldestMembership(); if (oldest !== undefined) { const selection = oldest.getTransport(oldest); - if (isLivekitTransport(selection)) local = ready(selection); + // TODO selection can be null if no transport is configured should we report an error? + if (selection && isLivekitTransport(selection)) + local = ready(selection); } } return { local, remote }; @@ -721,8 +723,8 @@ export class CallViewModel extends ViewModel { ), ); - private readonly userId = this.matrixRoom.client.getUserId(); - private readonly deviceId = this.matrixRoom.client.getDeviceId(); + private readonly userId = this.matrixRoom.client.getUserId()!; + private readonly deviceId = this.matrixRoom.client.getDeviceId()!; private readonly matrixConnected$ = this.scope.behavior( // To consider ourselves connected to MatrixRTC, we check the following: @@ -906,7 +908,11 @@ export class CallViewModel extends ViewModel { ], (memberships, _displaynames) => { const displaynameMap = new Map([ - ["local", this.matrixRoom.getMember(this.userId!)!.rawDisplayName], + [ + "local", + this.matrixRoom.getMember(this.userId)?.rawDisplayName ?? + this.userId, + ], ]); const room = this.matrixRoom; diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index 785cbe1ba..1b4d0cf0e 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -5,10 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - type CallMembership, - type MatrixRTCSession, -} from "matrix-js-sdk/lib/matrixrtc"; +import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc"; import { BehaviorSubject, of } from "rxjs"; import { vitest } from "vitest"; import { type RelationsContainer } from "matrix-js-sdk/lib/models/relations-container"; @@ -99,12 +96,12 @@ export function getBasicRTCSession( initialRtcMemberships, ); - const rtcSession = new MockRTCSession(matrixRoom).withMemberships( + const fakeRtcSession = new MockRTCSession(matrixRoom).withMemberships( rtcMemberships$, ); return { - rtcSession, + rtcSession: fakeRtcSession, matrixRoom, rtcMemberships$, }; @@ -137,7 +134,7 @@ export function getBasicCallViewModelEnvironment( // const remoteParticipants$ = of([aliceParticipant]); const vm = new CallViewModel( - rtcSession as unknown as MatrixRTCSession, + rtcSession.asMockedSession(), matrixRoom, mockMediaDevices({}), mockMuteStates(), diff --git a/src/utils/test.ts b/src/utils/test.ts index 508559c2b..cc0575325 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -6,7 +6,7 @@ Please see LICENSE in the repository root for full details. */ import { map, type Observable, of, type SchedulerLike } from "rxjs"; import { type RunHelpers, TestScheduler } from "rxjs/testing"; -import { expect, vi, vitest } from "vitest"; +import { expect, type MockedObject, vi, vitest } from "vitest"; import { type RoomMember, type Room as MatrixRoom, @@ -23,6 +23,7 @@ import { type SessionMembershipData, Status, type LivekitFocusSelection, + type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { type MembershipManagerEventHandlerMap } from "matrix-js-sdk/lib/matrixrtc/IMembershipManager"; import { @@ -193,7 +194,9 @@ export function mockRtcMembership( sender: typeof user === "string" ? user : user.userId, event_id: `$-ev-${randomUUID()}:example.org`, }); - return new CallMembership(event, data); + const cms = new CallMembership(event, data); + vi.mocked(cms).getTransport = vi.fn().mockReturnValue(fociPreferred[0]); + return cms; } // Maybe it'd be good to move this to matrix-js-sdk? Our testing needs are @@ -209,6 +212,7 @@ export function mockMatrixRoomMember( getMxcAvatarUrl(): string | undefined { return undefined; }, + rawDisplayName: rtcMembership.sender, ...member, } as RoomMember; } @@ -335,6 +339,22 @@ export class MockRTCSession extends TypedEventEmitter< RoomAndToDeviceEventsHandlerMap & MembershipManagerEventHandlerMap > { + public asMockedSession(): MockedObject { + const session = this as unknown as MockedObject; + + vi.mocked(session).reemitEncryptionKeys = vi + .fn<() => void>() + .mockReturnValue(undefined); + vi.mocked(session).resolveActiveFocus = vi + .fn<(member?: CallMembership) => Transport | undefined>() + .mockReturnValue(undefined); + vi.mocked(session).getOldestMembership = vi + .fn<() => CallMembership | undefined>() + .mockReturnValue(this.memberships[0]); + + return session; + } + public readonly statistics = { counters: {}, }; From 1ab081d6366f521532aaf2707bae3385cde2ee0d Mon Sep 17 00:00:00 2001 From: Valere Date: Fri, 10 Oct 2025 11:41:26 +0200 Subject: [PATCH 42/46] test: MISSING_MATRIX_RTC_FOCUS renamed as MISSING_MATRIX_RTC_TRANSPORT --- src/room/GroupCallErrorBoundary.test.tsx | 2 +- src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/room/GroupCallErrorBoundary.test.tsx b/src/room/GroupCallErrorBoundary.test.tsx index 223389249..869217107 100644 --- a/src/room/GroupCallErrorBoundary.test.tsx +++ b/src/room/GroupCallErrorBoundary.test.tsx @@ -106,7 +106,7 @@ test("should render the error page with link back to home", async () => { await screen.findByText("Call is not supported"); expect(screen.getByText(/Domain: example\.com/i)).toBeInTheDocument(); expect( - screen.getByText(/Error Code: MISSING_MATRIX_RTC_FOCUS/i), + screen.getByText(/Error Code: MISSING_MATRIX_RTC_TRANSPORT/i), ).toBeInTheDocument(); await screen.findByRole("button", { name: "Return to home screen" }); diff --git a/src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap b/src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap index ad4aff615..73a6df12c 100644 --- a/src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap +++ b/src/room/__snapshots__/GroupCallErrorBoundary.test.tsx.snap @@ -292,7 +292,7 @@ exports[`should have a close button in widget mode 1`] = ` Call is not supported

- The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_FOCUS). + The server is not configured to work with Element Call. Please contact your server admin (Domain: example.com, Error Code: MISSING_MATRIX_RTC_TRANSPORT).