Merge pull request #2583 from matrix-org/kegan/sync-v3

sliding sync bugfix: ensure history is treated as history and not live events

Author: kegsay

spec/integ/sliding-sync-sdk.spec.ts

@@ -28,6 +28,7 @@ import {
 import { SlidingSyncSdk } from "../../src/sliding-sync-sdk";
 import { SyncState } from "../../src/sync";
 import { IStoredClientOpts } from "../../src/client";
+import { logger } from "../../src/logger";
 
 describe("SlidingSyncSdk", () => {
     let client: MatrixClient = null;
@@ -372,6 +373,36 @@ describe("SlidingSyncSdk", () => {
                         gotRoom.getUnreadNotificationCount(NotificationCountType.Total),
                     ).toEqual(1);
                 });
+
+                // Regression test for a bug which caused the timeline entries to be out-of-order
+                // when the same room appears twice with different timeline limits. E.g appears in
+                // the list with timeline_limit:1 then appears again as a room subscription with
+                // timeline_limit:50
+                it("can return history with a larger timeline_limit", async () => {
+                    const timeline = data[roomA].timeline;
+                    const oldTimeline = [
+                        mkOwnEvent(EventType.RoomMessage, { body: "old event A" }),
+                        mkOwnEvent(EventType.RoomMessage, { body: "old event B" }),
+                        mkOwnEvent(EventType.RoomMessage, { body: "old event C" }),
+                        ...timeline,
+                    ];
+                    mockSlidingSync.emit(SlidingSyncEvent.RoomData, roomA, {
+                        timeline: oldTimeline,
+                        required_state: [],
+                        name: data[roomA].name,
+                        initial: true, // e.g requested via room subscription
+                    });
+                    const gotRoom = client.getRoom(roomA);
+                    expect(gotRoom).toBeDefined();
+
+                    logger.log("want:", oldTimeline.map((e) => (e.type + " : " + (e.content || {}).body)));
+                    logger.log("got:", gotRoom.getLiveTimeline().getEvents().map(
+                        (e) => (e.getType() + " : " + e.getContent().body)),
+                    );
+
+                    // we expect the timeline now to be oldTimeline (so the old events are in fact old)
+                    assertTimelineEvents(gotRoom.getLiveTimeline().getEvents(), oldTimeline);
+                });
             });
         });
     });

src/sliding-sync-sdk.ts

@@ -406,9 +406,51 @@ export class SlidingSyncSdk {
         // this helps large account to speed up faster
         // room::decryptCriticalEvent is in charge of decrypting all the events
         // required for a client to function properly
-        const timelineEvents = mapEvents(this.client, room.roomId, roomData.timeline, false);
+        let timelineEvents = mapEvents(this.client, room.roomId, roomData.timeline, false);
         const ephemeralEvents = []; // TODO this.mapSyncEventsFormat(joinObj.ephemeral);
 
+        // TODO: handle threaded / beacon events
+
+        if (roomData.initial) {
+            // we should not know about any of these timeline entries if this is a genuinely new room.
+            // If we do, then we've effectively done scrollback (e.g requesting timeline_limit: 1 for
+            // this room, then timeline_limit: 50).
+            const knownEvents = new Set<string>();
+            room.getLiveTimeline().getEvents().forEach((e) => {
+                knownEvents.add(e.getId());
+            });
+            // all unknown events BEFORE a known event must be scrollback e.g:
+            //       D E   <-- what we know
+            // A B C D E F <-- what we just received
+            // means:
+            // A B C       <-- scrollback
+            //       D E   <-- dupes
+            //           F <-- new event
+            // We bucket events based on if we have seen a known event yet.
+            const oldEvents: MatrixEvent[] = [];
+            const newEvents: MatrixEvent[] = [];
+            let seenKnownEvent = false;
+            for (let i = timelineEvents.length-1; i >= 0; i--) {
+                const recvEvent = timelineEvents[i];
+                if (knownEvents.has(recvEvent.getId())) {
+                    seenKnownEvent = true;
+                    continue; // don't include this event, it's a dupe
+                }
+                if (seenKnownEvent) {
+                    // old -> new
+                    oldEvents.push(recvEvent);
+                } else {
+                    // old -> new
+                    newEvents.unshift(recvEvent);
+                }
+            }
+            timelineEvents = newEvents;
+            if (oldEvents.length > 0) {
+                // old events are scrollback, insert them now
+                room.addEventsToTimeline(oldEvents, true, room.getLiveTimeline(), roomData.prev_batch);
+            }
+        }
+
         const encrypted = this.client.isRoomEncrypted(room.roomId);
         // we do this first so it's correct when any of the events fire
         if (roomData.notification_count != null) {