Support mid-call device changes

Signed-off-by: Šimon Brandner <[email protected]>

Author: SimonBrandner

src/client.ts

@@ -822,7 +822,7 @@ export class MatrixClient extends EventEmitter {
     protected checkTurnServersIntervalID: number;
     protected exportedOlmDeviceToImport: IOlmDevice;
     protected txnCtr = 0;
-    protected mediaHandler = new MediaHandler();
+    protected mediaHandler = new MediaHandler(this);
     protected pendingEventEncryption = new Map<string, Promise<void>>();
 
     constructor(opts: IMatrixClientCreateOpts) {

src/webrtc/mediaHandler.ts

@@ -18,29 +18,72 @@ limitations under the License.
 */
 
 import { logger } from "../logger";
+import { MatrixClient } from "../client";
+import { CallState } from "./call";
 
 export class MediaHandler {
     private audioInput: string;
     private videoInput: string;
-    private userMediaStreams: MediaStream[] = [];
-    private screensharingStreams: MediaStream[] = [];
+    private localUserMediaStream?: MediaStream;
+    public userMediaStreams: MediaStream[] = [];
+    public screensharingStreams: MediaStream[] = [];
+
+    constructor(private client: MatrixClient) { }
 
     /**
      * Set an audio input device to use for MatrixCalls
      * @param {string} deviceId the identifier for the device
      * undefined treated as unset
      */
-    public setAudioInput(deviceId: string): void {
+    public async setAudioInput(deviceId: string): Promise<void> {
+        if (this.audioInput === deviceId) return;
+
         this.audioInput = deviceId;
+        await this.updateLocalUsermediaStreams();
     }
 
     /**
      * Set a video input device to use for MatrixCalls
      * @param {string} deviceId the identifier for the device
      * undefined treated as unset
      */
-    public setVideoInput(deviceId: string): void {
+    public async setVideoInput(deviceId: string): Promise<void> {
+        if (this.videoInput === deviceId) return;
+
         this.videoInput = deviceId;
+        await this.updateLocalUsermediaStreams();
+    }
+
+    /**
+     * Requests new usermedia streams and replace the old ones
+     */
+    public async updateLocalUsermediaStreams(): Promise<void> {
+        if (this.userMediaStreams.length === 0) return;
+
+        const callMediaStreamParams: Map<string, { audio: boolean, video: boolean }> = new Map();
+        for (const call of this.client.callEventHandler.calls.values()) {
+            callMediaStreamParams.set(call.callId, {
+                audio: call.hasLocalUserMediaAudioTrack,
+                video: call.hasLocalUserMediaVideoTrack,
+            });
+        }
+
+        for (const stream of this.userMediaStreams) {
+            for (const track of stream.getTracks()) {
+                track.stop();
+            }
+        }
+
+        this.userMediaStreams = [];
+        this.localUserMediaStream = undefined;
+
+        for (const call of this.client.callEventHandler.calls.values()) {
+            if (call.state === CallState.Ended || !callMediaStreamParams.has(call.callId)) continue;
+
+            const { audio, video } = callMediaStreamParams.get(call.callId);
+            const stream = await this.getUserMediaStream(audio, video);
+            await call.updateLocalUsermediaStream(stream);
+        }
     }
 
     public async hasAudioDevice(): Promise<boolean> {
@@ -65,20 +108,40 @@ export class MediaHandler {
 
         let stream: MediaStream;
 
-        // Find a stream with matching tracks
-        const matchingStream = this.userMediaStreams.find((stream) => {
-            if (shouldRequestAudio !== (stream.getAudioTracks().length > 0)) return false;
-            if (shouldRequestVideo !== (stream.getVideoTracks().length > 0)) return false;
-            return true;
-        });
-
-        if (matchingStream) {
-            logger.log("Cloning user media stream", matchingStream.id);
-            stream = matchingStream.clone();
-        } else {
+        if (
+            !this.localUserMediaStream ||
+            (this.localUserMediaStream.getAudioTracks().length === 0 && shouldRequestAudio) ||
+            (this.localUserMediaStream.getVideoTracks().length === 0 && shouldRequestVideo)
+        ) {
             const constraints = this.getUserMediaContraints(shouldRequestAudio, shouldRequestVideo);
             logger.log("Getting user media with constraints", constraints);
             stream = await navigator.mediaDevices.getUserMedia(constraints);
+
+            for (const track of stream.getTracks()) {
+                const settings = track.getSettings();
+
+                if (track.kind === "audio") {
+                    this.audioInput = settings.deviceId;
+                } else if (track.kind === "video") {
+                    this.videoInput = settings.deviceId;
+                }
+            }
+
+            this.localUserMediaStream = stream;
+        } else {
+            stream = this.localUserMediaStream.clone();
+
+            if (!shouldRequestAudio) {
+                for (const track of stream.getAudioTracks()) {
+                    stream.removeTrack(track);
+                }
+            }
+
+            if (!shouldRequestVideo) {
+                for (const track of stream.getVideoTracks()) {
+                    stream.removeTrack(track);
+                }
+            }
         }
 
         if (reusable) {
@@ -103,6 +166,10 @@ export class MediaHandler {
             logger.debug("Splicing usermedia stream out stream array", mediaStream.id);
             this.userMediaStreams.splice(index, 1);
         }
+
+        if (this.localUserMediaStream === mediaStream) {
+            this.localUserMediaStream = undefined;
+        }
     }
 
     /**
@@ -174,6 +241,7 @@ export class MediaHandler {
 
         this.userMediaStreams = [];
         this.screensharingStreams = [];
+        this.localUserMediaStream = undefined;
     }
 
     private getUserMediaContraints(audio: boolean, video: boolean): MediaStreamConstraints {