Implement MSC3819: Allowing widgets to send/receive to-device messages (#8885)

* Implement MSC3819: Allowing widgets to send/receive to-device messages

* Don't change the room events and state events drivers

* Update to latest matrix-widget-api changes

* Support sending encrypted to-device messages

* Use queueToDevice for better reliability

* Update types for latest WidgetDriver changes

* Upgrade matrix-widget-api

* Add tests

* Test StopGapWidget

* Fix a potential memory leak

Author: robintown

package.json

@@ -94,7 +94,7 @@
     "matrix-encrypt-attachment": "^1.0.3",
     "matrix-events-sdk": "^0.0.1-beta.7",
     "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#develop",
-    "matrix-widget-api": "^0.1.0-beta.18",
+    "matrix-widget-api": "^1.0.0",
     "minimist": "^1.2.5",
     "opus-recorder": "^8.0.3",
     "pako": "^2.0.3",

src/stores/widgets/StopGapWidget.ts

@@ -33,6 +33,7 @@ import {
     WidgetKind,
 } from "matrix-widget-api";
 import { EventEmitter } from "events";
+import { MatrixClient } from "matrix-js-sdk/src/client";
 import { MatrixEvent, MatrixEventEvent } from "matrix-js-sdk/src/models/event";
 import { logger } from "matrix-js-sdk/src/logger";
 import { ClientEvent } from "matrix-js-sdk/src/client";
@@ -148,6 +149,7 @@ export class ElementWidget extends Widget {
 }
 
 export class StopGapWidget extends EventEmitter {
+    private client: MatrixClient;
     private messaging: ClientWidgetApi;
     private mockWidget: ElementWidget;
     private scalarToken: string;
@@ -157,12 +159,13 @@ export class StopGapWidget extends EventEmitter {
 
     constructor(private appTileProps: IAppTileProps) {
         super();
-        let app = appTileProps.app;
+        this.client = MatrixClientPeg.get();
 
+        let app = appTileProps.app;
         // Backwards compatibility: not all old widgets have a creatorUserId
         if (!app.creatorUserId) {
             app = objectShallowClone(app); // clone to prevent accidental mutation
-            app.creatorUserId = MatrixClientPeg.get().getUserId();
+            app.creatorUserId = this.client.getUserId();
         }
 
         this.mockWidget = new ElementWidget(app);
@@ -203,7 +206,7 @@ export class StopGapWidget extends EventEmitter {
         const fromCustomisation = WidgetVariableCustomisations?.provideVariables?.() ?? {};
         const defaults: ITemplateParams = {
             widgetRoomId: this.roomId,
-            currentUserId: MatrixClientPeg.get().getUserId(),
+            currentUserId: this.client.getUserId(),
             userDisplayName: OwnProfileStore.instance.displayName,
             userHttpAvatarUrl: OwnProfileStore.instance.getHttpAvatarUrl(),
             clientId: ELEMENT_CLIENT_ID,
@@ -260,8 +263,10 @@ export class StopGapWidget extends EventEmitter {
      */
     public startMessaging(iframe: HTMLIFrameElement): any {
         if (this.started) return;
+
         const allowedCapabilities = this.appTileProps.whitelistCapabilities || [];
         const driver = new StopGapWidgetDriver(allowedCapabilities, this.mockWidget, this.kind, this.roomId);
+
         this.messaging = new ClientWidgetApi(this.mockWidget, iframe, driver);
         this.messaging.on("preparing", () => this.emit("preparing"));
         this.messaging.on("ready", () => this.emit("ready"));
@@ -302,7 +307,7 @@ export class StopGapWidget extends EventEmitter {
         // Populate the map of "read up to" events for this widget with the current event in every room.
         // This is a bit inefficient, but should be okay. We do this for all rooms in case the widget
         // requests timeline capabilities in other rooms down the road. It's just easier to manage here.
-        for (const room of MatrixClientPeg.get().getRooms()) {
+        for (const room of this.client.getRooms()) {
             // Timelines are most recent last
             const events = room.getLiveTimeline()?.getEvents() || [];
             const roomEvent = events[events.length - 1];
@@ -311,8 +316,9 @@ export class StopGapWidget extends EventEmitter {
         }
 
         // Attach listeners for feeding events - the underlying widget classes handle permissions for us
-        MatrixClientPeg.get().on(ClientEvent.Event, this.onEvent);
-        MatrixClientPeg.get().on(MatrixEventEvent.Decrypted, this.onEventDecrypted);
+        this.client.on(ClientEvent.Event, this.onEvent);
+        this.client.on(MatrixEventEvent.Decrypted, this.onEventDecrypted);
+        this.client.on(ClientEvent.ToDeviceEvent, this.onToDeviceEvent);
 
         this.messaging.on(`action:${WidgetApiFromWidgetAction.UpdateAlwaysOnScreen}`,
             (ev: CustomEvent<IStickyActionRequest>) => {
@@ -363,7 +369,7 @@ export class StopGapWidget extends EventEmitter {
 
                     // noinspection JSIgnoredPromiseFromCall
                     IntegrationManagers.sharedInstance().getPrimaryManager().open(
-                        MatrixClientPeg.get().getRoom(RoomViewStore.instance.getRoomId()),
+                        this.client.getRoom(RoomViewStore.instance.getRoomId()),
                         `type_${integType}`,
                         integId,
                     );
@@ -428,14 +434,13 @@ export class StopGapWidget extends EventEmitter {
         WidgetMessagingStore.instance.stopMessaging(this.mockWidget, this.roomId);
         this.messaging = null;
 
-        if (MatrixClientPeg.get()) {
-            MatrixClientPeg.get().off(ClientEvent.Event, this.onEvent);
-            MatrixClientPeg.get().off(MatrixEventEvent.Decrypted, this.onEventDecrypted);
-        }
+        this.client.off(ClientEvent.Event, this.onEvent);
+        this.client.off(MatrixEventEvent.Decrypted, this.onEventDecrypted);
+        this.client.off(ClientEvent.ToDeviceEvent, this.onToDeviceEvent);
     }
 
     private onEvent = (ev: MatrixEvent) => {
-        MatrixClientPeg.get().decryptEventIfNeeded(ev);
+        this.client.decryptEventIfNeeded(ev);
         if (ev.isBeingDecrypted() || ev.isDecryptionFailure()) return;
         this.feedEvent(ev);
     };
@@ -445,6 +450,12 @@ export class StopGapWidget extends EventEmitter {
         this.feedEvent(ev);
     };
 
+    private onToDeviceEvent = async (ev: MatrixEvent) => {
+        await this.client.decryptEventIfNeeded(ev);
+        if (ev.isDecryptionFailure()) return;
+        await this.messaging.feedToDevice(ev.getEffectiveEvent(), ev.isEncrypted());
+    };
+
     private feedEvent(ev: MatrixEvent) {
         if (!this.messaging) return;
 
@@ -465,7 +476,7 @@ export class StopGapWidget extends EventEmitter {
 
             // Timelines are most recent last, so reverse the order and limit ourselves to 100 events
             // to avoid overusing the CPU.
-            const timeline = MatrixClientPeg.get().getRoom(ev.getRoomId()).getLiveTimeline();
+            const timeline = this.client.getRoom(ev.getRoomId()).getLiveTimeline();
             const events = arrayFastClone(timeline.getEvents()).reverse().slice(0, 100);
 
             for (const timelineEvent of events) {

src/stores/widgets/StopGapWidgetDriver.ts

@@ -20,6 +20,7 @@ import {
     IOpenIDCredentials,
     IOpenIDUpdate,
     ISendEventDetails,
+    IRoomEvent,
     MatrixCapabilities,
     OpenIDRequestState,
     SimpleObservable,
@@ -182,6 +183,49 @@ export class StopGapWidgetDriver extends WidgetDriver {
         return { roomId, eventId: r.event_id };
     }
 
+    public async sendToDevice(
+        eventType: string,
+        encrypted: boolean,
+        contentMap: { [userId: string]: { [deviceId: string]: object } },
+    ): Promise<void> {
+        const client = MatrixClientPeg.get();
+
+        if (encrypted) {
+            const deviceInfoMap = await client.crypto.deviceList.downloadKeys(Object.keys(contentMap), false);
+
+            await Promise.all(
+                Object.entries(contentMap).flatMap(([userId, userContentMap]) =>
+                    Object.entries(userContentMap).map(async ([deviceId, content]) => {
+                        if (deviceId === "*") {
+                            // Send the message to all devices we have keys for
+                            await client.encryptAndSendToDevices(
+                                Object.values(deviceInfoMap[userId]).map(deviceInfo => ({
+                                    userId, deviceInfo,
+                                })),
+                                content,
+                            );
+                        } else {
+                            // Send the message to a specific device
+                            await client.encryptAndSendToDevices(
+                                [{ userId, deviceInfo: deviceInfoMap[userId][deviceId] }],
+                                content,
+                            );
+                        }
+                    }),
+                ),
+            );
+        } else {
+            await client.queueToDevice({
+                eventType,
+                batch: Object.entries(contentMap).flatMap(([userId, userContentMap]) =>
+                    Object.entries(userContentMap).map(([deviceId, content]) =>
+                        ({ userId, deviceId, payload: content }),
+                    ),
+                ),
+            });
+        }
+    }
+
     private pickRooms(roomIds: (string | Symbols.AnyRoom)[] = null): Room[] {
         const client = MatrixClientPeg.get();
         if (!client) throw new Error("Not attached to a client");
@@ -197,7 +241,7 @@ export class StopGapWidgetDriver extends WidgetDriver {
         msgtype: string | undefined,
         limitPerRoom: number,
         roomIds: (string | Symbols.AnyRoom)[] = null,
-    ): Promise<object[]> {
+    ): Promise<IRoomEvent[]> {
         limitPerRoom = limitPerRoom > 0 ? Math.min(limitPerRoom, Number.MAX_SAFE_INTEGER) : Number.MAX_SAFE_INTEGER; // relatively arbitrary
 
         const rooms = this.pickRooms(roomIds);
@@ -224,7 +268,7 @@ export class StopGapWidgetDriver extends WidgetDriver {
         stateKey: string | undefined,
         limitPerRoom: number,
         roomIds: (string | Symbols.AnyRoom)[] = null,
-    ): Promise<object[]> {
+    ): Promise<IRoomEvent[]> {
         limitPerRoom = limitPerRoom > 0 ? Math.min(limitPerRoom, Number.MAX_SAFE_INTEGER) : Number.MAX_SAFE_INTEGER; // relatively arbitrary
 
         const rooms = this.pickRooms(roomIds);

src/widgets/CapabilityText.tsx

@@ -17,6 +17,7 @@ limitations under the License.
 import {
     Capability,
     EventDirection,
+    EventKind,
     getTimelineRoomIDFromCapability,
     isTimelineCapability,
     isTimelineCapabilityFor,
@@ -134,7 +135,7 @@ export class CapabilityText {
     };
 
     private static bylineFor(eventCap: WidgetEventCapability): TranslatedString {
-        if (eventCap.isState) {
+        if (eventCap.kind === EventKind.State) {
             return !eventCap.keyStr
                 ? _t("with an empty state key")
                 : _t("with state key %(stateKey)s", { stateKey: eventCap.keyStr });
@@ -143,6 +144,8 @@ export class CapabilityText {
     }
 
     public static for(capability: Capability, kind: WidgetKind): TranslatedCapabilityText {
+        // TODO: Support MSC3819 (to-device capabilities)
+
         // First see if we have a super simple line of text to provide back
         if (CapabilityText.simpleCaps[capability]) {
             const textForKind = CapabilityText.simpleCaps[capability];
@@ -184,13 +187,13 @@ export class CapabilityText {
             // Special case room messages so they show up a bit cleaner to the user. Result is
             // effectively "Send images" instead of "Send messages... of type images" if we were
             // to handle the msgtype nuances in this function.
-            if (!eventCap.isState && eventCap.eventType === EventType.RoomMessage) {
+            if (eventCap.kind === EventKind.Event && eventCap.eventType === EventType.RoomMessage) {
                 return CapabilityText.forRoomMessageCap(eventCap, kind);
             }
 
             // See if we have a static line of text to provide for the given event type and
             // direction. The hope is that we do for common event types for friendlier copy.
-            const evSendRecv = eventCap.isState
+            const evSendRecv = eventCap.kind === EventKind.State
                 ? CapabilityText.stateSendRecvCaps
                 : CapabilityText.nonStateSendRecvCaps;
             if (evSendRecv[eventCap.eventType]) {

test/stores/widgets/StopGapWidget-test.ts

@@ -0,0 +1,70 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { mocked, MockedObject } from "jest-mock";
+import { MatrixClient, ClientEvent } from "matrix-js-sdk/src/client";
+import { ClientWidgetApi } from "matrix-widget-api";
+
+import { stubClient, mkRoom, mkEvent } from "../../test-utils";
+import { MatrixClientPeg } from "../../../src/MatrixClientPeg";
+import { StopGapWidget } from "../../../src/stores/widgets/StopGapWidget";
+
+jest.mock("matrix-widget-api/lib/ClientWidgetApi");
+
+describe("StopGapWidget", () => {
+    let client: MockedObject<MatrixClient>;
+    let widget: StopGapWidget;
+    let messaging: MockedObject<ClientWidgetApi>;
+
+    beforeEach(() => {
+        stubClient();
+        client = mocked(MatrixClientPeg.get());
+
+        widget = new StopGapWidget({
+            app: {
+                id: "test",
+                creatorUserId: "@alice:example.org",
+                type: "example",
+                url: "https://example.org",
+            },
+            room: mkRoom(client, "!1:example.org"),
+            userId: "@alice:example.org",
+            creatorUserId: "@alice:example.org",
+            waitForIframeLoad: true,
+            userWidget: false,
+        });
+        // Start messaging without an iframe, since ClientWidgetApi is mocked
+        widget.startMessaging(null as unknown as HTMLIFrameElement);
+        messaging = mocked(mocked(ClientWidgetApi).mock.instances[0]);
+    });
+
+    afterEach(() => {
+        widget.stopMessaging();
+    });
+
+    it("feeds incoming to-device messages to the widget", async () => {
+        const event = mkEvent({
+            event: true,
+            type: "org.example.foo",
+            user: "@alice:example.org",
+            content: { hello: "world" },
+        });
+
+        client.emit(ClientEvent.ToDeviceEvent, event);
+        await Promise.resolve(); // flush promises
+        expect(messaging.feedToDevice).toHaveBeenCalledWith(event.getEffectiveEvent(), false);
+    });
+});