From a285fe05fd7bfcca3953976e7b3356dc39b020a7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Feb 2018 13:45:30 +0000 Subject: [PATCH 001/178] Add timeout to ResponseCache of /public_rooms --- synapse/handlers/room_list.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index fc507cef36..878db0db1e 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -46,9 +46,12 @@ class RoomListHandler(BaseHandler): def __init__(self, hs): super(RoomListHandler, self).__init__(hs) - self.response_cache = ResponseCache(hs, "room_list") - self.remote_response_cache = ResponseCache(hs, "remote_room_list", - timeout_ms=30 * 1000) + self.response_cache = ResponseCache( + hs, "room_list", timeout_ms=10 * 60 * 1000, + ) + self.remote_response_cache = ResponseCache( + hs, "remote_room_list", timeout_ms=30 * 1000, + ) def get_local_public_room_list(self, limit=None, since_token=None, search_filter=None, From 66dcbf47a36b5ca0e88d4658578d6fb5e6dbd910 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Feb 2018 13:47:14 +0000 Subject: [PATCH 002/178] Disable auto search for prefixes in event search --- synapse/storage/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index f0fa5d7631..ef4f587d8c 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -722,7 +722,7 @@ def _parse_query(database_engine, search_term): results = re.findall(r"([\w\-]+)", search_term, re.UNICODE) if isinstance(database_engine, PostgresEngine): - return " & ".join(result + ":*" for result in results) + return " & ".join(result for result in results) elif isinstance(database_engine, Sqlite3Engine): return " & ".join(result + "*" for result in results) else: From 43bb12e640ce1563541b246bffe09870d43892ea Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 5 May 2017 11:02:34 +0100 Subject: [PATCH 003/178] Disable presence This reverts commit 0ebd376a53bb75ade6d65db2d716478758c2c9f0 and disables presence a bit more --- synapse/app/synchrotron.py | 1 + synapse/handlers/initial_sync.py | 1 + synapse/handlers/presence.py | 3 +++ synapse/handlers/sync.py | 2 +- synapse/rest/client/v1/presence.py | 2 +- tests/rest/client/v1/test_rooms.py | 12 +++++++----- 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 82f06ea185..3cade3c68e 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -116,6 +116,7 @@ def __init__(self, hs): logger.info("Presence process_id is %r", self.process_id) def send_user_sync(self, user_id, is_syncing, last_sync_ms): + return self.hs.get_tcp_replication().send_user_sync(user_id, is_syncing, last_sync_ms) def mark_as_coming_online(self, user_id): diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index 71af86fe21..5242309c1e 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -373,6 +373,7 @@ def _room_initial_sync_joined(self, user_id, room_id, pagin_config, @defer.inlineCallbacks def get_presence(): + defer.returnValue([]) states = yield presence_handler.get_states( [m.user_id for m in room_members], as_event=True, diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 7fe568132f..c7336020a2 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -391,6 +391,7 @@ def bump_presence_active_time(self, user): """We've seen the user do something that indicates they're interacting with the app. """ + return user_id = user.to_string() bump_active_time_counter.inc() @@ -420,6 +421,7 @@ def user_syncing(self, user_id, affect_presence=True): Useful for streams that are not associated with an actual client that is being used by a user. """ + affect_presence = False if affect_presence: curr_sync = self.user_to_num_current_syncs.get(user_id, 0) self.user_to_num_current_syncs[user_id] = curr_sync + 1 @@ -465,6 +467,7 @@ def get_currently_syncing_users(self): Returns: set(str): A set of user_id strings. """ + return set() syncing_user_ids = { user_id for user_id, count in self.user_to_num_current_syncs.items() if count diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 51ec727df0..1e8a50514b 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -620,7 +620,7 @@ def generate_sync_result(self, sync_config, since_token=None, full_state=False): since_token is None and sync_config.filter_collection.blocks_all_presence() ) - if not block_all_presence_data: + if False and not block_all_presence_data: yield self._generate_sync_entry_for_presence( sync_result_builder, newly_joined_rooms, newly_joined_users ) diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py index 647994bd53..a975666d12 100644 --- a/synapse/rest/client/v1/presence.py +++ b/synapse/rest/client/v1/presence.py @@ -83,7 +83,7 @@ def on_PUT(self, request, user_id): except Exception: raise SynapseError(400, "Unable to parse state") - yield self.presence_handler.set_state(user, state) + # yield self.presence_handler.set_state(user, state) defer.returnValue((200, {})) diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 61d737725b..5bc0ba1934 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -984,11 +984,13 @@ def test_initial_sync(self): self.assertTrue("presence" in response) - presence_by_user = { - e["content"]["user_id"]: e for e in response["presence"] - } - self.assertTrue(self.user_id in presence_by_user) - self.assertEquals("m.presence", presence_by_user[self.user_id]["type"]) + # presence is turned off on hotfixes + + # presence_by_user = { + # e["content"]["user_id"]: e for e in response["presence"] + # } + # self.assertTrue(self.user_id in presence_by_user) + # self.assertEquals("m.presence", presence_by_user[self.user_id]["type"]) class RoomMessageListTestCase(RestTestCase): From e5537cf98333b428dbc481ed443daed2f0cfa074 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 23 May 2017 10:48:13 +0100 Subject: [PATCH 004/178] Limit concurrent AS joins --- synapse/handlers/room_member.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index f930e939e8..5e351df8aa 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -31,7 +31,7 @@ ) from synapse.api.errors import AuthError, SynapseError, Codes from synapse.types import UserID, RoomID -from synapse.util.async import Linearizer +from synapse.util.async import Linearizer, Limiter from synapse.util.distributor import user_left_room, user_joined_room @@ -68,6 +68,7 @@ def __init__(self, hs): self.event_creation_hander = hs.get_event_creation_handler() self.member_linearizer = Linearizer(name="member") + self.member_limiter = Limiter(3) self.clock = hs.get_clock() self.spam_checker = hs.get_spam_checker() @@ -241,18 +242,23 @@ def update_membership( ): key = (room_id,) - with (yield self.member_linearizer.queue(key)): - result = yield self._update_membership( - requester, - target, - room_id, - action, - txn_id=txn_id, - remote_room_hosts=remote_room_hosts, - third_party_signed=third_party_signed, - ratelimit=ratelimit, - content=content, - ) + as_id = object() + if requester.app_service: + as_id = requester.app_service.id + + with (yield self.member_limiter.queue(as_id)): + with (yield self.member_linearizer.queue(key)): + result = yield self._update_membership( + requester, + target, + room_id, + action, + txn_id=txn_id, + remote_room_hosts=remote_room_hosts, + third_party_signed=third_party_signed, + ratelimit=ratelimit, + content=content, + ) defer.returnValue(result) From e6b1ea3eb20dcf60e1d460716bb658f8935de8d6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Feb 2018 13:52:29 +0000 Subject: [PATCH 005/178] Disable presence in txn queue --- synapse/federation/transaction_queue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index d72b057e28..b9d0639a1b 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -303,6 +303,7 @@ def send_presence(self, states): Args: states (list(UserPresenceState)) """ + return # First we queue up the new presence by user ID, so multiple presence # updates in quick successtion are correctly handled From 1766a5fdc09da970b44477690b8f1825e1265fb4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Feb 2018 13:53:02 +0000 Subject: [PATCH 006/178] Increase MAX_EVENTS_BEHIND for replication clients --- synapse/replication/tcp/streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/replication/tcp/streams.py b/synapse/replication/tcp/streams.py index 4c60bf79f9..07d99dd63c 100644 --- a/synapse/replication/tcp/streams.py +++ b/synapse/replication/tcp/streams.py @@ -33,7 +33,7 @@ logger = logging.getLogger(__name__) -MAX_EVENTS_BEHIND = 10000 +MAX_EVENTS_BEHIND = 500000 EventStreamRow = namedtuple("EventStreamRow", ( From a79823e64b0aa271e085d3fa8e29724b4b752cfe Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Feb 2018 12:17:14 +0000 Subject: [PATCH 007/178] Add dummy presence REST handler to frontend proxy The handler no-ops all requests as presence is disabled. --- synapse/app/frontend_proxy.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index d2bae4ad03..e6db28333f 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -37,6 +37,7 @@ from synapse.replication.slave.storage.devices import SlavedDeviceStore from synapse.replication.slave.storage.registration import SlavedRegistrationStore from synapse.replication.tcp.client import ReplicationClientHandler +from synapse.rest.client.v1.base import ClientV1RestServlet, client_path_patterns from synapse.rest.client.v2_alpha._base import client_v2_patterns from synapse.server import HomeServer from synapse.storage.engines import create_engine @@ -50,6 +51,35 @@ logger = logging.getLogger("synapse.app.frontend_proxy") +class PresenceStatusStubServlet(ClientV1RestServlet): + PATTERNS = client_path_patterns("/presence/(?P[^/]*)/status") + + def __init__(self, hs): + super(PresenceStatusStubServlet, self).__init__(hs) + self.http_client = hs.get_simple_http_client() + self.auth = hs.get_auth() + self.main_uri = hs.config.worker_main_http_uri + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + # Pass through the auth headers, if any, in case the access token + # is there. + auth_headers = request.requestHeaders.getRawHeaders("Authorization", []) + headers = { + "Authorization": auth_headers, + } + result = yield self.http_client.get_json( + self.main_uri + request.uri, + headers=headers, + ) + defer.returnValue((200, result)) + + @defer.inlineCallbacks + def on_PUT(self, request, user_id): + yield self.auth.get_user_by_req(request) + defer.returnValue((200, {})) + + class KeyUploadServlet(RestServlet): PATTERNS = client_v2_patterns("/keys/upload(/(?P[^/]+))?$") @@ -136,6 +166,7 @@ def _listen_http(self, listener_config): elif name == "client": resource = JsonResource(self, canonical_json=False) KeyUploadServlet(self).register(resource) + PresenceStatusStubServlet(self).register(resource) resources.update({ "/_matrix/client/r0": resource, "/_matrix/client/unstable": resource, From 637387483348928c368963bf91dab6213ef336bb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Feb 2018 12:23:16 +0000 Subject: [PATCH 008/178] Move event sending to end in shutdown room admin api --- synapse/rest/client/v1/admin.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index ddaedb2a8c..e4036eaa0c 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -312,17 +312,6 @@ def on_POST(self, request, room_id): ) new_room_id = info["room_id"] - yield self.event_creation_handler.create_and_send_nonmember_event( - room_creator_requester, - { - "type": "m.room.message", - "content": {"body": message, "msgtype": "m.text"}, - "room_id": new_room_id, - "sender": new_room_user_id, - }, - ratelimit=False, - ) - requester_user_id = requester.user.to_string() logger.info("Shutting down room %r", room_id) @@ -360,6 +349,17 @@ def on_POST(self, request, room_id): kicked_users.append(user_id) + yield self.event_creation_handler.create_and_send_nonmember_event( + room_creator_requester, + { + "type": "m.room.message", + "content": {"body": message, "msgtype": "m.text"}, + "room_id": new_room_id, + "sender": new_room_user_id, + }, + ratelimit=False, + ) + aliases_for_room = yield self.store.get_aliases_for_room(room_id) yield self.store.update_aliases_for_room( From 7a1406d144ca32b83a050e947f423cda8e964cd6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Feb 2018 12:23:59 +0000 Subject: [PATCH 009/178] Prefill client_ip_last_seen in replication --- synapse/replication/slave/storage/client_ips.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/replication/slave/storage/client_ips.py b/synapse/replication/slave/storage/client_ips.py index 352c9a2aa8..fedf7a3188 100644 --- a/synapse/replication/slave/storage/client_ips.py +++ b/synapse/replication/slave/storage/client_ips.py @@ -42,6 +42,8 @@ def insert_client_ip(self, user_id, access_token, ip, user_agent, device_id): if last_seen is not None and (now - last_seen) < LAST_SEEN_GRANULARITY: return + self.client_ip_last_seen.prefill(key, now) + self.hs.get_tcp_replication().send_user_ip( user_id, access_token, ip, user_agent, device_id, now ) From 8f8ea91eefcc43c5ac24e85b14a86af4da53e6e0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Feb 2018 12:24:26 +0000 Subject: [PATCH 010/178] Bump LAST_SEEN_GRANULARITY in client_ips --- synapse/storage/client_ips.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 968d2fed22..30d3541cf7 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -30,7 +30,7 @@ # Number of msec of granularity to store the user IP 'last seen' time. Smaller # times give more inserts into the database even for readonly API hits # 120 seconds == 2 minutes -LAST_SEEN_GRANULARITY = 120 * 1000 +LAST_SEEN_GRANULARITY = 10 * 60 * 1000 class ClientIpStore(background_updates.BackgroundUpdateStore): From fae708c0e8c35930f1172322b7c0e9f0b1b3f9a4 Mon Sep 17 00:00:00 2001 From: hera Date: Thu, 12 Oct 2017 10:41:46 +0000 Subject: [PATCH 011/178] Disable auth on room_members for now because the moznet bridge is broken (https://github.com/matrix-org/matrix-appservice-irc/issues/506) --- synapse/handlers/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7b9946ab91..b194f113ce 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -390,7 +390,7 @@ def get_joined_members(self, requester, room_id): # If this is an AS, double check that they are allowed to see the members. # This can either be because the AS user is in the room or becuase there # is a user in the room that the AS is "interested in" - if requester.app_service and user_id not in users_with_profile: + if False and requester.app_service and user_id not in users_with_profile: for uid in users_with_profile: if requester.app_service.is_interested_in_user(uid): break From 1031bd25f8f37413f4101b0a049a598f1d95f8d3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 24 Nov 2017 00:38:44 +0000 Subject: [PATCH 012/178] Avoid doing presence updates on replication reconnect Presence is supposed to be disabled on matrix.org, so we shouldn't send a load of USER_SYNC commands every time the synchrotron reconnects to the master. --- synapse/app/synchrotron.py | 2 ++ synapse/handlers/presence.py | 1 + 2 files changed, 3 insertions(+) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 3cade3c68e..67a2d2b7db 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -214,6 +214,8 @@ def process_replication_rows(self, token, rows): yield self.notify_from_replication(states, stream_id) def get_currently_syncing_users(self): + # presence is disabled on matrix.org, so we return the empty set + return set() return [ user_id for user_id, count in iteritems(self.user_to_num_current_syncs) if count > 0 diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index c7336020a2..95953efb08 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -467,6 +467,7 @@ def get_currently_syncing_users(self): Returns: set(str): A set of user_id strings. """ + # presence is disabled on matrix.org, so we return the empty set return set() syncing_user_ids = { user_id for user_id, count in self.user_to_num_current_syncs.items() From e21c312e16b33ebcf8f031757986c601fe0320c4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 21 Feb 2018 21:09:33 +0000 Subject: [PATCH 013/178] Actuall set cache factors in workers --- synapse/app/synctl.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 56ae086128..8c525459c7 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -108,7 +108,7 @@ def stop(pidfile, app): Worker = collections.namedtuple("Worker", [ - "app", "configfile", "pidfile", "cache_factor" + "app", "configfile", "pidfile", "cache_factor", "cache_factors", ]) @@ -215,6 +215,10 @@ def main(): or pidfile ) worker_cache_factor = worker_config.get("synctl_cache_factor") or cache_factor + worker_cache_factors = ( + worker_config.get("synctl_cache_factors") + or cache_factors + ) daemonize = worker_config.get("daemonize") or config.get("daemonize") assert daemonize, "Main process must have daemonize set to true" @@ -230,8 +234,10 @@ def main(): assert worker_daemonize, "In config %r: expected '%s' to be True" % ( worker_configfile, "worker_daemonize") worker_cache_factor = worker_config.get("synctl_cache_factor") + worker_cache_factors = worker_config.get("synctl_cache_factors", {}) workers.append(Worker( worker_app, worker_configfile, worker_pidfile, worker_cache_factor, + worker_cache_factors, )) action = options.action @@ -266,15 +272,19 @@ def main(): start(configfile) for worker in workers: + env = os.environ.copy() + if worker.cache_factor: os.environ["SYNAPSE_CACHE_FACTOR"] = str(worker.cache_factor) + for cache_name, factor in worker.cache_factors.iteritems(): + os.environ["SYNAPSE_CACHE_FACTOR_" + cache_name.upper()] = str(factor) + start_worker(worker.app, configfile, worker.configfile) - if cache_factor: - os.environ["SYNAPSE_CACHE_FACTOR"] = str(cache_factor) - else: - os.environ.pop("SYNAPSE_CACHE_FACTOR", None) + # Reset env back to the original + os.environ.clear() + os.environ.update(env) if __name__ == "__main__": From 0ca2857baab2d241a740a6ba5e8ed36d17756338 Mon Sep 17 00:00:00 2001 From: hera Date: Wed, 28 Feb 2018 00:35:03 +0000 Subject: [PATCH 014/178] increase sync cache to 2 minutes to give synchrotrons being hammered by repeating initial /syncs to get more chance to actually complete and avoid a DoS --- synapse/handlers/sync.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 1e8a50514b..8df66b0fcf 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -32,6 +32,7 @@ logger = logging.getLogger(__name__) +SYNC_RESPONSE_CACHE_MS = 2 * 60 * 1000 SyncConfig = collections.namedtuple("SyncConfig", [ "user", @@ -178,7 +179,9 @@ def __init__(self, hs): self.presence_handler = hs.get_presence_handler() self.event_sources = hs.get_event_sources() self.clock = hs.get_clock() - self.response_cache = ResponseCache(hs, "sync") + self.response_cache = ResponseCache( + hs, "sync", timeout_ms=SYNC_RESPONSE_CACHE_MS, + ) self.state = hs.get_state_handler() def wait_for_sync_for_user(self, sync_config, since_token=None, timeout=0, From e18378c3e26a6464c14a61a30d201ecd97b7ef4e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 4 Apr 2018 16:14:02 +0100 Subject: [PATCH 015/178] Increase member limiter to 20 Let's see if this makes the bridges go faster, or if it kills the synapse master. --- synapse/handlers/room_member.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 5e351df8aa..a5ea2c3ae2 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -68,7 +68,7 @@ def __init__(self, hs): self.event_creation_hander = hs.get_event_creation_handler() self.member_linearizer = Linearizer(name="member") - self.member_limiter = Limiter(3) + self.member_limiter = Limiter(20) self.clock = hs.get_clock() self.spam_checker = hs.get_spam_checker() From 80786d5cafe3bfe7637116c2208455384f65e97b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 4 Apr 2018 17:30:02 +0100 Subject: [PATCH 016/178] Logging for get_users_in_room --- synapse/storage/roommember.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 829cc4a207..3032f10185 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -68,6 +68,7 @@ def get_hosts_in_room(self, room_id, cache_context): defer.returnValue(hosts) @cached(max_entries=100000, iterable=True) + @defer.inlineCallbacks def get_users_in_room(self, room_id): def f(txn): sql = ( @@ -81,7 +82,14 @@ def f(txn): txn.execute(sql, (room_id, Membership.JOIN,)) return [to_ascii(r[0]) for r in txn] - return self.runInteraction("get_users_in_room", f) + start_time = self._clock.time_msec() + result = yield self.runInteraction("get_users_in_room", f) + end_time = self._clock.time_msec() + logger.info( + "Fetched room membership for %s (%i users) in %i ms", + room_id, len(result), end_time - start_time, + ) + defer.returnValue(result) @cached() def get_invited_rooms_for_user(self, user_id): From ce9d0b1d0c0b362194138093004a7bc7ce147cc2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 4 Apr 2018 23:20:30 +0100 Subject: [PATCH 017/178] Fix earlier logging patch `@cached` doesn't work on decorated functions, because it uses inspection on the target to calculate the number of arguments. --- synapse/storage/roommember.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 3032f10185..682c637c04 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -67,8 +67,7 @@ def get_hosts_in_room(self, room_id, cache_context): hosts = frozenset(get_domain_from_id(user_id) for user_id in user_ids) defer.returnValue(hosts) - @cached(max_entries=100000, iterable=True) - @defer.inlineCallbacks + @cachedInlineCallbacks(max_entries=100000, iterable=True) def get_users_in_room(self, room_id): def f(txn): sql = ( From 463e7c27097a4e48315365c6fc6cbe6bfe4493c8 Mon Sep 17 00:00:00 2001 From: hera Date: Tue, 24 Apr 2018 11:30:54 +0000 Subject: [PATCH 018/178] Lower member limiter --- synapse/handlers/room_member.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index a5ea2c3ae2..edc33e466a 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -68,7 +68,7 @@ def __init__(self, hs): self.event_creation_hander = hs.get_event_creation_handler() self.member_linearizer = Linearizer(name="member") - self.member_limiter = Limiter(20) + self.member_limiter = Limiter(10) self.clock = hs.get_clock() self.spam_checker = hs.get_spam_checker() From 9e38981ae47d03467a954c3c540c51b567f6e50b Mon Sep 17 00:00:00 2001 From: aphrodite Date: Mon, 4 Jun 2018 14:24:28 +0000 Subject: [PATCH 019/178] Send HTTP pushes direct to http-priv rather than via clouldflare (This is a heinous hack that ought to be made more generic and pushed back to develop) --- synapse/push/httppusher.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index bf7ff74a1a..e0ccbfa360 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -329,7 +329,12 @@ def dispatch_push(self, event, tweaks, badge): if not notification_dict: defer.returnValue([]) try: - resp = yield self.http_client.post_json_get_json(self.url, notification_dict) + url = self.url.replace( + "https://matrix.org/_matrix/push/v1/notify", + "http://http-priv.matrix.org/_matrix/push/v1/notify", + ) + + resp = yield self.http_client.post_json_get_json(url, notification_dict) except Exception: logger.warn( "Failed to push event %s to %s", From 7a32fa01013b1faeace51c71cb98e8a981e79b3b Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 26 Jun 2018 10:52:52 +0100 Subject: [PATCH 020/178] Fix error on deleting users pending deactivation Use simple_delete instead of simple_delete_one as commented --- synapse/storage/registration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 9c9cf46e7f..0d18f6d869 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -623,7 +623,9 @@ def del_user_pending_deactivation(self, user_id): Removes the given user to the table of users who need to be parted from all the rooms they're in, effectively marking that user as fully deactivated. """ - return self._simple_delete_one( + # XXX: This should be simple_delete_one but we failed to put a unique index on + # the table, so somehow duplicate entries have ended up in it. + return self._simple_delete( "users_pending_deactivation", keyvalues={ "user_id": user_id, From ca21957b8a4942e829050834297afb290338c120 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 2 Jul 2018 13:56:08 +0100 Subject: [PATCH 021/178] Timeout membership requests after 90s This is a hacky fix to try and stop in flight requests from building up --- synapse/handlers/room_member.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index edc33e466a..e0950288df 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -246,8 +246,22 @@ def update_membership( if requester.app_service: as_id = requester.app_service.id + then = self.clock.time_msec() + with (yield self.member_limiter.queue(as_id)): + diff = self.clock.time_msec() - then + + if diff > 90 * 1000: + # haproxy would have timed the request out anyway... + raise SynapseError(504, "took to long to process") + with (yield self.member_linearizer.queue(key)): + diff = self.clock.time_msec() - then + + if diff > 90 * 1000: + # haproxy would have timed the request out anyway... + raise SynapseError(504, "took to long to process") + result = yield self._update_membership( requester, target, From bf425e533ed678c6e22e2d8655263f4ab6d1edc6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Jul 2018 10:11:09 +0100 Subject: [PATCH 022/178] Fix PEP8 --- synapse/handlers/room_member.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index e0950288df..7d509f9e08 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -249,11 +249,11 @@ def update_membership( then = self.clock.time_msec() with (yield self.member_limiter.queue(as_id)): - diff = self.clock.time_msec() - then + diff = self.clock.time_msec() - then - if diff > 90 * 1000: - # haproxy would have timed the request out anyway... - raise SynapseError(504, "took to long to process") + if diff > 90 * 1000: + # haproxy would have timed the request out anyway... + raise SynapseError(504, "took to long to process") with (yield self.member_linearizer.queue(key)): diff = self.clock.time_msec() - then From 27eb4c45cda26a6d4d02351a8fa7a3cd07c4ab7f Mon Sep 17 00:00:00 2001 From: hera Date: Mon, 23 Jul 2018 15:16:36 +0000 Subject: [PATCH 023/178] Lower hacky timeout for member limiter --- synapse/handlers/room_member.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 5b91105176..4872796a31 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -251,14 +251,14 @@ def update_membership( with (yield self.member_limiter.queue(as_id)): diff = self.clock.time_msec() - then - if diff > 90 * 1000: + if diff > 80 * 1000: # haproxy would have timed the request out anyway... raise SynapseError(504, "took to long to process") with (yield self.member_linearizer.queue(key)): diff = self.clock.time_msec() - then - if diff > 90 * 1000: + if diff > 80 * 1000: # haproxy would have timed the request out anyway... raise SynapseError(504, "took to long to process") From e43eb47c5f73288188677cc53ffffadb3a62a34e Mon Sep 17 00:00:00 2001 From: hera Date: Mon, 23 Jul 2018 15:22:47 +0000 Subject: [PATCH 024/178] Fixup limiter --- synapse/handlers/room_member.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 4872796a31..6184737cd4 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -30,7 +30,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.errors import AuthError, Codes, SynapseError from synapse.types import RoomID, UserID -from synapse.util.async import Limiter, Linearizer +from synapse.util.async import Linearizer from synapse.util.distributor import user_joined_room, user_left_room logger = logging.getLogger(__name__) @@ -66,7 +66,7 @@ def __init__(self, hs): self.event_creation_hander = hs.get_event_creation_handler() self.member_linearizer = Linearizer(name="member") - self.member_limiter = Limiter(10) + self.member_limiter = Linearizer(max_count=10, name="member_as_limiter") self.clock = hs.get_clock() self.spam_checker = hs.get_spam_checker() From 6e15b5debe3014ff513233a5faa1ad052ad921c2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 26 Sep 2018 13:25:52 +0100 Subject: [PATCH 025/178] Revert "Actuall set cache factors in workers" This reverts commit e21c312e16b33ebcf8f031757986c601fe0320c4. --- synapse/app/synctl.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/synapse/app/synctl.py b/synapse/app/synctl.py index 356e5cb6a7..d658f967ba 100755 --- a/synapse/app/synctl.py +++ b/synapse/app/synctl.py @@ -111,7 +111,7 @@ def stop(pidfile, app): Worker = collections.namedtuple("Worker", [ - "app", "configfile", "pidfile", "cache_factor", "cache_factors", + "app", "configfile", "pidfile", "cache_factor" ]) @@ -218,10 +218,6 @@ def main(): or pidfile ) worker_cache_factor = worker_config.get("synctl_cache_factor") or cache_factor - worker_cache_factors = ( - worker_config.get("synctl_cache_factors") - or cache_factors - ) daemonize = worker_config.get("daemonize") or config.get("daemonize") assert daemonize, "Main process must have daemonize set to true" @@ -237,10 +233,8 @@ def main(): assert worker_daemonize, "In config %r: expected '%s' to be True" % ( worker_configfile, "worker_daemonize") worker_cache_factor = worker_config.get("synctl_cache_factor") - worker_cache_factors = worker_config.get("synctl_cache_factors", {}) workers.append(Worker( worker_app, worker_configfile, worker_pidfile, worker_cache_factor, - worker_cache_factors, )) action = options.action @@ -275,19 +269,15 @@ def main(): start(configfile) for worker in workers: - env = os.environ.copy() - if worker.cache_factor: os.environ["SYNAPSE_CACHE_FACTOR"] = str(worker.cache_factor) - for cache_name, factor in worker.cache_factors.iteritems(): - os.environ["SYNAPSE_CACHE_FACTOR_" + cache_name.upper()] = str(factor) - start_worker(worker.app, configfile, worker.configfile) - # Reset env back to the original - os.environ.clear() - os.environ.update(env) + if cache_factor: + os.environ["SYNAPSE_CACHE_FACTOR"] = str(cache_factor) + else: + os.environ.pop("SYNAPSE_CACHE_FACTOR", None) if __name__ == "__main__": From 5c0c4b4079b43e6929cca19de63b580cb6863f06 Mon Sep 17 00:00:00 2001 From: hera Date: Thu, 8 Nov 2018 11:03:08 +0000 Subject: [PATCH 026/178] Fix encoding error for consent form on python3 The form was rendering this as "b'01234....'". -- richvdh --- synapse/rest/consent/consent_resource.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index e0f7de5d5c..8009b7ff1c 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -160,7 +160,9 @@ def _async_render_GET(self, request): try: self._render_template( request, "%s.html" % (version,), - user=username, userhmac=userhmac, version=version, + user=username, + userhmac=userhmac.decode('ascii'), + version=version, has_consented=has_consented, public_version=public_version, ) except TemplateNotFound: From 9accd63a387013081ef0035b919bd2c32910c752 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 30 Nov 2018 12:04:38 +0000 Subject: [PATCH 027/178] Initial patch from Erik --- synapse/rest/client/v1/room.py | 25 ++++++++++++++++++++ synapse/storage/events_worker.py | 40 ++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index fcfe7857f6..218dbb93ff 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -825,6 +825,30 @@ def on_GET(self, request): defer.returnValue((200, {"joined_rooms": list(room_ids)})) +class TimestampLookupRestServlet(ClientV1RestServlet): + PATTERNS = client_path_patterns("/rooms/(?P[^/]*)/timestamp_to_event$") + + def __init__(self, hs): + super(TimestampLookupRestServlet, self).__init__(hs) + self.store = hs.get_datastore() + + @defer.inlineCallbacks + def on_GET(self, request, room_id): + requester = yield self.auth.get_user_by_req(request) + yield self.auth.check_joined_room(room_id, requester.user.to_string()) + + timestamp = parse_integer(request, "ts") + thread_id = parse_integer(request, "thread_id", 0) + + event_id = yield self.store.get_event_for_timestamp( + room_id, thread_id, timestamp, + ) + + defer.returnValue((200, { + "event_id": event_id, + })) + + def register_txn_path(servlet, regex_string, http_server, with_get=False): """Registers a transaction-based path. @@ -874,6 +898,7 @@ def register_servlets(hs, http_server): JoinedRoomsRestServlet(hs).register(http_server) RoomEventServlet(hs).register(http_server) RoomEventContextServlet(hs).register(http_server) + TimestampLookupRestServlet(hs).register(http_server) def register_deprecated_servlets(hs, http_server): diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py index a8326f5296..e4d8562396 100644 --- a/synapse/storage/events_worker.py +++ b/synapse/storage/events_worker.py @@ -526,3 +526,43 @@ def f(txn): return res return self.runInteraction("get_rejection_reasons", f) + + def get_event_for_timestamp(self, room_id, thread_id, timestamp): + sql_template = """ + SELECT event_id, origin_server_ts FROM events + WHERE + origin_server_ts %s ? + AND room_id = ? + AND thread_id = ? + ORDER BY origin_server_ts + LIMIT 1; + """ + + def f(txn): + txn.execute(sql_template % ("<=",), (timestamp, room_id, thread_id)) + row = txn.fetchone() + if row: + event_id_before, ts_before = row + else: + event_id_before, ts_before = None, None + + txn.execute(sql_template % (">=",), (timestamp, room_id, thread_id)) + row = txn.fetchone() + if row: + event_id_after, ts_after = row + else: + event_id_after, ts_after = None, None + + if event_id_before and event_id_before: + # Return the closest one + if (timestamp - ts_before) < (ts_after - timestamp): + return event_id_before + else: + return event_id_after + + if event_id_before: + return event_id_before + + return event_id_after + + return self.runInteraction("get_event_for_timestamp", f) From 080025e5331d4d51943590800adb9c6c5257a047 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 30 Nov 2018 12:09:33 +0000 Subject: [PATCH 028/178] Fix buglet and remove thread_id stuff --- synapse/rest/client/v1/room.py | 3 +-- synapse/storage/events_worker.py | 9 ++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 218dbb93ff..f296257b11 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -838,10 +838,9 @@ def on_GET(self, request, room_id): yield self.auth.check_joined_room(room_id, requester.user.to_string()) timestamp = parse_integer(request, "ts") - thread_id = parse_integer(request, "thread_id", 0) event_id = yield self.store.get_event_for_timestamp( - room_id, thread_id, timestamp, + room_id, timestamp, ) defer.returnValue((200, { diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py index e4d8562396..c5fee97ac9 100644 --- a/synapse/storage/events_worker.py +++ b/synapse/storage/events_worker.py @@ -527,33 +527,32 @@ def f(txn): return self.runInteraction("get_rejection_reasons", f) - def get_event_for_timestamp(self, room_id, thread_id, timestamp): + def get_event_for_timestamp(self, room_id, timestamp): sql_template = """ SELECT event_id, origin_server_ts FROM events WHERE origin_server_ts %s ? AND room_id = ? - AND thread_id = ? ORDER BY origin_server_ts LIMIT 1; """ def f(txn): - txn.execute(sql_template % ("<=",), (timestamp, room_id, thread_id)) + txn.execute(sql_template % ("<=",), (timestamp, room_id)) row = txn.fetchone() if row: event_id_before, ts_before = row else: event_id_before, ts_before = None, None - txn.execute(sql_template % (">=",), (timestamp, room_id, thread_id)) + txn.execute(sql_template % (">=",), (timestamp, room_id)) row = txn.fetchone() if row: event_id_after, ts_after = row else: event_id_after, ts_after = None, None - if event_id_before and event_id_before: + if event_id_before and event_id_after: # Return the closest one if (timestamp - ts_before) < (ts_after - timestamp): return event_id_before From c4074e4ab67e53705029657147e847ee08a429b5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 3 Dec 2018 10:15:39 +0000 Subject: [PATCH 029/178] Revert "Merge branch 'rav/timestamp_patch' into matrix-org-hotfixes" This reverts commit 7960e814e5386b32821ef51915e609091e6e1569, reversing changes made to 3dd704ee9af836b9b3e50daf1017a3d65c9ebaec. We no longer need this; please redo it as a proper MSC & synapse PR if you want to keep it... --- synapse/rest/client/v1/room.py | 24 -------------------- synapse/storage/events_worker.py | 39 -------------------------------- 2 files changed, 63 deletions(-) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index f296257b11..fcfe7857f6 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -825,29 +825,6 @@ def on_GET(self, request): defer.returnValue((200, {"joined_rooms": list(room_ids)})) -class TimestampLookupRestServlet(ClientV1RestServlet): - PATTERNS = client_path_patterns("/rooms/(?P[^/]*)/timestamp_to_event$") - - def __init__(self, hs): - super(TimestampLookupRestServlet, self).__init__(hs) - self.store = hs.get_datastore() - - @defer.inlineCallbacks - def on_GET(self, request, room_id): - requester = yield self.auth.get_user_by_req(request) - yield self.auth.check_joined_room(room_id, requester.user.to_string()) - - timestamp = parse_integer(request, "ts") - - event_id = yield self.store.get_event_for_timestamp( - room_id, timestamp, - ) - - defer.returnValue((200, { - "event_id": event_id, - })) - - def register_txn_path(servlet, regex_string, http_server, with_get=False): """Registers a transaction-based path. @@ -897,7 +874,6 @@ def register_servlets(hs, http_server): JoinedRoomsRestServlet(hs).register(http_server) RoomEventServlet(hs).register(http_server) RoomEventContextServlet(hs).register(http_server) - TimestampLookupRestServlet(hs).register(http_server) def register_deprecated_servlets(hs, http_server): diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py index c5fee97ac9..a8326f5296 100644 --- a/synapse/storage/events_worker.py +++ b/synapse/storage/events_worker.py @@ -526,42 +526,3 @@ def f(txn): return res return self.runInteraction("get_rejection_reasons", f) - - def get_event_for_timestamp(self, room_id, timestamp): - sql_template = """ - SELECT event_id, origin_server_ts FROM events - WHERE - origin_server_ts %s ? - AND room_id = ? - ORDER BY origin_server_ts - LIMIT 1; - """ - - def f(txn): - txn.execute(sql_template % ("<=",), (timestamp, room_id)) - row = txn.fetchone() - if row: - event_id_before, ts_before = row - else: - event_id_before, ts_before = None, None - - txn.execute(sql_template % (">=",), (timestamp, room_id)) - row = txn.fetchone() - if row: - event_id_after, ts_after = row - else: - event_id_after, ts_after = None, None - - if event_id_before and event_id_after: - # Return the closest one - if (timestamp - ts_before) < (ts_after - timestamp): - return event_id_before - else: - return event_id_after - - if event_id_before: - return event_id_before - - return event_id_after - - return self.runInteraction("get_event_for_timestamp", f) From 046d731fbdf0f7058395fc0c691c9029c1fcb4c9 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 15 Jan 2019 21:07:12 +0000 Subject: [PATCH 030/178] limit remote device lists to 1000 entries per user --- synapse/handlers/device.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 9e017116a9..6f80a7dce9 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -532,6 +532,20 @@ def _handle_device_updates(self, user_id): stream_id = result["stream_id"] devices = result["devices"] + + # Emergency hack to prevent DoS from + # @bot:oliviervandertoorn.nl and @bot:matrix-beta.igalia.com + # on Jan 15 2019: only store the most recent 1000 devices for + # a given user. (We assume we receive them in chronological + # order, which is dubious given _get_e2e_device_keys_txn does + # not explicitly order its results). Otherwise it can take + # longer than 60s to persist the >100K devices, at which point + # the internal replication request to handle the + # m.device_list_update EDU times out, causing the remote + # server to retry the transaction and thus DoS synapse master + # CPU and DB. + devices = devices[-1000:] + yield self.store.update_remote_device_list_cache( user_id, devices, stream_id, ) From 482d06774ac456943fb7e519a78431c82da305ca Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 15 Jan 2019 21:38:07 +0000 Subject: [PATCH 031/178] don't store remote device lists if they have more than 10K devices --- synapse/handlers/device.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 6f80a7dce9..5bca62418e 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -533,18 +533,19 @@ def _handle_device_updates(self, user_id): stream_id = result["stream_id"] devices = result["devices"] - # Emergency hack to prevent DoS from - # @bot:oliviervandertoorn.nl and @bot:matrix-beta.igalia.com - # on Jan 15 2019: only store the most recent 1000 devices for - # a given user. (We assume we receive them in chronological - # order, which is dubious given _get_e2e_device_keys_txn does - # not explicitly order its results). Otherwise it can take - # longer than 60s to persist the >100K devices, at which point - # the internal replication request to handle the - # m.device_list_update EDU times out, causing the remote - # server to retry the transaction and thus DoS synapse master - # CPU and DB. - devices = devices[-1000:] + # If the remote server has more than ~10000 devices for this user + # we assume that something is going horribly wrong (e.g. a bot + # that logs in and creates a new device every time it tries to + # send a message). Maintaining lots of devices per user in the + # cache can cause serious performance issues as if this request + # takes more than 60s to complete, internal replication from the + # inbound federation worker to the synapse master may time out + # causing the inbound federation to fail and causing the remote + # server to retry, causing a DoS. So in this scenario we give + # up on storing the total list of devices and only handle the + # delta instead. + if len(devices) > 10000: + devices = [] yield self.store.update_remote_device_list_cache( user_id, devices, stream_id, From b4796d18149948fed4b864b61f655ecfe068d4b3 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 15 Jan 2019 21:46:29 +0000 Subject: [PATCH 032/178] drop the limit to 1K as e2e will be hosed beyond that point anyway --- synapse/handlers/device.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 5bca62418e..a93dfd1d63 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -533,7 +533,7 @@ def _handle_device_updates(self, user_id): stream_id = result["stream_id"] devices = result["devices"] - # If the remote server has more than ~10000 devices for this user + # If the remote server has more than ~1000 devices for this user # we assume that something is going horribly wrong (e.g. a bot # that logs in and creates a new device every time it tries to # send a message). Maintaining lots of devices per user in the @@ -544,7 +544,7 @@ def _handle_device_updates(self, user_id): # server to retry, causing a DoS. So in this scenario we give # up on storing the total list of devices and only handle the # delta instead. - if len(devices) > 10000: + if len(devices) > 1000: devices = [] yield self.store.update_remote_device_list_cache( From 5cb15c0443c6258056b9dcaa8c28bcce5fbb312e Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 15 Jan 2019 22:10:44 +0000 Subject: [PATCH 033/178] warn if we ignore device lists --- synapse/handlers/device.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index a93dfd1d63..8955cde4ed 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -545,6 +545,10 @@ def _handle_device_updates(self, user_id): # up on storing the total list of devices and only handle the # delta instead. if len(devices) > 1000: + logger.warn( + "Ignoring device list snapshot for %s as it has >1K devs (%d)", + user_id, len(devices) + ) devices = [] yield self.store.update_remote_device_list_cache( From ef7865e2f2a8cd353a332244e81de8db0370b6d2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 23 Jan 2019 14:17:21 +0000 Subject: [PATCH 034/178] Track user_dir current event stream position --- synapse/handlers/user_directory.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 3c40999338..55e0332549 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -19,6 +19,7 @@ from twisted.internet import defer +from prometheus_client import Gauge from synapse.api.constants import EventTypes, JoinRules, Membership from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.roommember import ProfileInfo @@ -27,6 +28,8 @@ logger = logging.getLogger(__name__) +# Expose event stream processing position +event_processing_position = Gauge("event_stream_processing_position", "Currently processed up to position in the event stream") class UserDirectoryHandler(object): """Handles querying of and keeping updated the user_directory. @@ -163,6 +166,10 @@ def _unsafe_process(self): yield self._handle_deltas(deltas) self.pos = deltas[-1]["stream_id"] + + # Expose current event processing position to prometheus + event_processing_position.set(self.pos) + yield self.store.update_user_directory_stream_pos(self.pos) @defer.inlineCallbacks From 77dfe51aba3190a146556cff8a32a66a607a5ff9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 23 Jan 2019 14:21:19 +0000 Subject: [PATCH 035/178] Name metric consistently --- synapse/handlers/user_directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 55e0332549..6df58060da 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -29,7 +29,7 @@ logger = logging.getLogger(__name__) # Expose event stream processing position -event_processing_position = Gauge("event_stream_processing_position", "Currently processed up to position in the event stream") +event_processing_position = Gauge("synapse_user_dir_event_stream_position", "Currently processed up to position in the event stream") class UserDirectoryHandler(object): """Handles querying of and keeping updated the user_directory. From 7da659dd6d1cadac233855d0b5a2269ba0ac280b Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 23 Jan 2019 15:01:09 +0000 Subject: [PATCH 036/178] Use existing stream position counter metric --- synapse/handlers/user_directory.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 6df58060da..197a826daa 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -19,7 +19,6 @@ from twisted.internet import defer -from prometheus_client import Gauge from synapse.api.constants import EventTypes, JoinRules, Membership from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.roommember import ProfileInfo @@ -28,9 +27,6 @@ logger = logging.getLogger(__name__) -# Expose event stream processing position -event_processing_position = Gauge("synapse_user_dir_event_stream_position", "Currently processed up to position in the event stream") - class UserDirectoryHandler(object): """Handles querying of and keeping updated the user_directory. @@ -168,7 +164,8 @@ def _unsafe_process(self): self.pos = deltas[-1]["stream_id"] # Expose current event processing position to prometheus - event_processing_position.set(self.pos) + synapse.metrics.event_processing_positions.labels( + "user_dir").set(self.pos) yield self.store.update_user_directory_stream_pos(self.pos) From 4777836b832d715cb2b49879ca93868cdf1ab727 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 23 Jan 2019 15:26:03 +0000 Subject: [PATCH 037/178] Fix missing synapse metrics import --- synapse/handlers/user_directory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 197a826daa..a9f062df4b 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -14,6 +14,7 @@ # limitations under the License. import logging +import synapse.metrics from six import iteritems From decb5698b3dabd0c42ec4ce085bb4bb580de8f3a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 31 Jan 2019 00:23:58 +0000 Subject: [PATCH 038/178] Break infinite loop on redaction in v3 rooms --- synapse/storage/events_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py index 57dae324c7..dc09a85087 100644 --- a/synapse/storage/events_worker.py +++ b/synapse/storage/events_worker.py @@ -177,7 +177,7 @@ def _get_events(self, event_ids, check_redacted=True, # Starting in room version v3, some redactions need to be rechecked if we # didn't have the redacted event at the time, so we recheck on read # instead. - if not allow_rejected and entry.event.type == EventTypes.Redaction: + if not allow_rejected and entry.event.type == EventTypes.Redaction and False: if entry.event.internal_metadata.need_to_check_redaction(): orig = yield self.get_event( entry.event.redacts, From e97c1df30c23eaa3ee96ea7b761eb5107af35e6a Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 31 Jan 2019 13:26:38 +0000 Subject: [PATCH 039/178] remove slow code on userdir (#4534) --- synapse/handlers/user_directory.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 71ea2e3cee..f5c3ba23a6 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -338,24 +338,6 @@ def _handle_deltas(self, deltas): public_value=Membership.JOIN, ) - if change is False: - # Need to check if the server left the room entirely, if so - # we might need to remove all the users in that room - is_in_room = yield self.store.is_host_joined( - room_id, self.server_name, - ) - if not is_in_room: - logger.info("Server left room: %r", room_id) - # Fetch all the users that we marked as being in user - # directory due to being in the room and then check if - # need to remove those users or not - user_ids = yield self.store.get_users_in_dir_due_to_room(room_id) - for user_id in user_ids: - yield self._handle_remove_user(room_id, user_id) - return - else: - logger.debug("Server is still in room: %r", room_id) - is_support = yield self.store.is_support_user(state_key) if not is_support: if change is None: From 6c232a69dfa23dff154e49918b3133669e28d062 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 31 Jan 2019 18:43:49 +0000 Subject: [PATCH 040/178] Revert "Break infinite loop on redaction in v3 rooms" We've got a better fix of this now. This reverts commit decb5698b3dabd0c42ec4ce085bb4bb580de8f3a. --- synapse/storage/events_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py index dc09a85087..57dae324c7 100644 --- a/synapse/storage/events_worker.py +++ b/synapse/storage/events_worker.py @@ -177,7 +177,7 @@ def _get_events(self, event_ids, check_redacted=True, # Starting in room version v3, some redactions need to be rechecked if we # didn't have the redacted event at the time, so we recheck on read # instead. - if not allow_rejected and entry.event.type == EventTypes.Redaction and False: + if not allow_rejected and entry.event.type == EventTypes.Redaction: if entry.event.internal_metadata.need_to_check_redaction(): orig = yield self.get_event( entry.event.redacts, From f5bafd70f4d641b24303958b650efcf7dc18ab92 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 04:34:23 +1100 Subject: [PATCH 041/178] add cache remover endpoint and wire it up --- synapse/replication/http/__init__.py | 3 ++- synapse/rest/client/v2_alpha/register.py | 11 +++++++++++ synapse/storage/registration.py | 11 +++++++---- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py index 19f214281e..f46a580be2 100644 --- a/synapse/replication/http/__init__.py +++ b/synapse/replication/http/__init__.py @@ -14,7 +14,7 @@ # limitations under the License. from synapse.http.server import JsonResource -from synapse.replication.http import federation, membership, send_event +from synapse.replication.http import federation, membership, send_event, registration REPLICATION_PREFIX = "/_synapse/replication" @@ -28,3 +28,4 @@ def register_servlets(self, hs): send_event.register_servlets(hs, self) membership.register_servlets(hs, self) federation.register_servlets(hs, self) + registration.register_servlets(hs, self) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 7f812b8209..35319f7a79 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -24,6 +24,7 @@ import synapse import synapse.types +from synapse.replication.http.registration import RegistrationUserCacheInvalidationServlet from synapse.api.constants import LoginType from synapse.api.errors import Codes, SynapseError, UnrecognizedRequestError from synapse.config.server import is_threepid_reserved @@ -193,6 +194,10 @@ def __init__(self, hs): self.device_handler = hs.get_device_handler() self.macaroon_gen = hs.get_macaroon_generator() + self._invalidate_caches_client = ( + RegistrationUserCacheInvalidationServlet.make_client(hs) + ) + @interactive_auth_handler @defer.inlineCallbacks def on_POST(self, request): @@ -266,6 +271,9 @@ def on_POST(self, request): # == Shared Secret Registration == (e.g. create new user scripts) if 'mac' in body: + if self.hs.config.worker_app: + raise SynapseError(403, "Not available at this endpoint") + # FIXME: Should we really be determining if this is shared secret # auth based purely on the 'mac' key? result = yield self._do_shared_secret_registration( @@ -456,6 +464,9 @@ def on_POST(self, request): ) yield self.registration_handler.post_consent_actions(registered_user_id) + if self.hs.config.worker_app: + self._invalidate_caches_client(registered_user_id) + defer.returnValue((200, return_dict)) def on_OPTIONS(self, _): diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index c9e11c3135..8b4554f6af 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -146,6 +146,7 @@ class RegistrationStore(RegistrationWorkerStore, def __init__(self, db_conn, hs): super(RegistrationStore, self).__init__(db_conn, hs) + self.hs = hs self.clock = hs.get_clock() self.register_background_index_update( @@ -321,10 +322,12 @@ def _register( (user_id_obj.localpart, create_profile_with_displayname) ) - self._invalidate_cache_and_stream( - txn, self.get_user_by_id, (user_id,) - ) - txn.call_after(self.is_guest.invalidate, (user_id,)) + # Don't invalidate here, it will be done through replication to the worker. + if not self.hs.config.worker_app: + self._invalidate_cache_and_stream( + txn, self.get_user_by_id, (user_id,) + ) + txn.call_after(self.is_guest.invalidate, (user_id,)) def get_users_by_id_case_insensitive(self, user_id): """Gets users that match user_id case insensitively. From 7b5c04312efd0829855a45b19839971c55f81599 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 04:35:27 +1100 Subject: [PATCH 042/178] isort --- synapse/handlers/user_directory.py | 1 - synapse/replication/http/__init__.py | 2 +- synapse/rest/client/v2_alpha/register.py | 4 +++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index f5c3ba23a6..0dacd9e357 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -14,7 +14,6 @@ # limitations under the License. import logging -import synapse.metrics from six import iteritems diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py index f46a580be2..3807d2ac6f 100644 --- a/synapse/replication/http/__init__.py +++ b/synapse/replication/http/__init__.py @@ -14,7 +14,7 @@ # limitations under the License. from synapse.http.server import JsonResource -from synapse.replication.http import federation, membership, send_event, registration +from synapse.replication.http import federation, membership, registration, send_event REPLICATION_PREFIX = "/_synapse/replication" diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 35319f7a79..ca9a850817 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -24,7 +24,6 @@ import synapse import synapse.types -from synapse.replication.http.registration import RegistrationUserCacheInvalidationServlet from synapse.api.constants import LoginType from synapse.api.errors import Codes, SynapseError, UnrecognizedRequestError from synapse.config.server import is_threepid_reserved @@ -34,6 +33,9 @@ parse_json_object_from_request, parse_string, ) +from synapse.replication.http.registration import ( + RegistrationUserCacheInvalidationServlet, +) from synapse.util.msisdn import phone_number_to_msisdn from synapse.util.ratelimitutils import FederationRateLimiter from synapse.util.threepids import check_3pid_allowed From ce5f3b1ba5cefe5d453bd82c06faa8eaa336137f Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 04:35:58 +1100 Subject: [PATCH 043/178] add all the files --- synapse/replication/http/registration.py | 70 ++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 synapse/replication/http/registration.py diff --git a/synapse/replication/http/registration.py b/synapse/replication/http/registration.py new file mode 100644 index 0000000000..5030bde499 --- /dev/null +++ b/synapse/replication/http/registration.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from twisted.internet import defer + +from synapse.events import event_type_from_format_version +from synapse.events.snapshot import EventContext +from synapse.http.servlet import parse_json_object_from_request +from synapse.replication.http._base import ReplicationEndpoint +from synapse.types import Requester, UserID +from synapse.util.metrics import Measure + +logger = logging.getLogger(__name__) + + +class RegistrationUserCacheInvalidationServlet(ReplicationEndpoint): + """ + Invalidate the caches that a registration usually invalidates. + + Request format: + + POST /_synapse/replication/fed_query/:fed_cleanup_room/:txn_id + + {} + """ + + NAME = "reg_invalidate_user_caches" + PATH_ARGS = ("user_id",) + + def __init__(self, hs): + super(RegistrationUserCacheInvalidationServlet, self).__init__(hs) + self.store = hs.get_datastore() + + @staticmethod + def _serialize_payload(user_id, args): + """ + Args: + user_id (str) + """ + return {} + + @defer.inlineCallbacks + def _handle_request(self, request, user_id): + + def invalidate(txn): + self.store._invalidate_cache_and_stream( + txn, self.store.get_user_by_id, (user_id,) + ) + txn.call_after(self.store.is_guest.invalidate, (user_id,)) + + yield self.store.runInteraction("user_invalidate_caches", invalidate) + defer.returnValue((200, {})) + + +def register_servlets(hs, http_server): + RegistrationUserCacheInvalidationServlet(hs).register(http_server) From d9235b9e29c88f8992b64904382e35905749351b Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 04:39:49 +1100 Subject: [PATCH 044/178] fix appservice, add to frontend proxy --- synapse/app/frontend_proxy.py | 6 ++++++ synapse/rest/client/v2_alpha/register.py | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index d5b954361d..5d450718c6 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -39,8 +39,12 @@ from synapse.replication.tcp.client import ReplicationClientHandler from synapse.rest.client.v1.base import ClientV1RestServlet, client_path_patterns from synapse.rest.client.v2_alpha._base import client_v2_patterns +from synapse.rest.client.v2_alpha.register import ( + register_servlets as register_registration_servlets, +) from synapse.server import HomeServer from synapse.storage.engines import create_engine +from synapse.storage.registration import RegistrationStore from synapse.util.httpresourcetree import create_resource_tree from synapse.util.logcontext import LoggingContext from synapse.util.manhole import manhole @@ -141,6 +145,7 @@ class FrontendProxySlavedStore( SlavedClientIpStore, SlavedApplicationServiceStore, SlavedRegistrationStore, + RegistrationStore, BaseSlavedStore, ): pass @@ -161,6 +166,7 @@ def _listen_http(self, listener_config): elif name == "client": resource = JsonResource(self, canonical_json=False) KeyUploadServlet(self).register(resource) + register_registration_servlets(self, resource) # If presence is disabled, use the stub servlet that does # not allow sending presence diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index ca9a850817..5ca9421a07 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -479,6 +479,10 @@ def _do_appservice_registration(self, username, as_token, body): user_id = yield self.registration_handler.appservice_register( username, as_token ) + + if self.hs.config.worker_app: + self._invalidate_caches_client(registered_user_id) + defer.returnValue((yield self._create_registration_details(user_id, body))) @defer.inlineCallbacks From e4381ed5144ba16faae0f2aedf07927344e69a4f Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 04:42:04 +1100 Subject: [PATCH 045/178] pep8 --- synapse/replication/http/registration.py | 5 ----- synapse/rest/client/v2_alpha/register.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/synapse/replication/http/registration.py b/synapse/replication/http/registration.py index 5030bde499..513dafab39 100644 --- a/synapse/replication/http/registration.py +++ b/synapse/replication/http/registration.py @@ -17,12 +17,7 @@ from twisted.internet import defer -from synapse.events import event_type_from_format_version -from synapse.events.snapshot import EventContext -from synapse.http.servlet import parse_json_object_from_request from synapse.replication.http._base import ReplicationEndpoint -from synapse.types import Requester, UserID -from synapse.util.metrics import Measure logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 5ca9421a07..9f750f7cf5 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -481,7 +481,7 @@ def _do_appservice_registration(self, username, as_token, body): ) if self.hs.config.worker_app: - self._invalidate_caches_client(registered_user_id) + self._invalidate_caches_client(user_id) defer.returnValue((yield self._create_registration_details(user_id, body))) From 84528e4fb2e191a065e51ae978755719b8b2ae71 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 04:49:09 +1100 Subject: [PATCH 046/178] cleanup --- synapse/rest/client/v2_alpha/register.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 9f750f7cf5..da9ebd540d 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -467,7 +467,7 @@ def on_POST(self, request): yield self.registration_handler.post_consent_actions(registered_user_id) if self.hs.config.worker_app: - self._invalidate_caches_client(registered_user_id) + yield self._invalidate_caches_client(registered_user_id) defer.returnValue((200, return_dict)) @@ -481,7 +481,7 @@ def _do_appservice_registration(self, username, as_token, body): ) if self.hs.config.worker_app: - self._invalidate_caches_client(user_id) + yield self._invalidate_caches_client(user_id) defer.returnValue((yield self._create_registration_details(user_id, body))) From d95252c01fa26f84fe995914f12260b38c200a28 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 05:08:58 +1100 Subject: [PATCH 047/178] use a device replication thingy --- synapse/replication/http/device.py | 64 ++++++++++++++++++++++++ synapse/replication/http/registration.py | 2 +- synapse/rest/client/v2_alpha/register.py | 31 +++++++++--- 3 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 synapse/replication/http/device.py diff --git a/synapse/replication/http/device.py b/synapse/replication/http/device.py new file mode 100644 index 0000000000..4855ba098f --- /dev/null +++ b/synapse/replication/http/device.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from twisted.internet import defer + +from synapse.http.servlet import parse_json_object_from_request +from synapse.replication.http._base import ReplicationEndpoint + +logger = logging.getLogger(__name__) + + +class CheckDeviceRegisteredServlet(ReplicationEndpoint): + """ + Check a device is registered. + + """ + + NAME = "device_check_registered" + PATH_ARGS = ("user_id") + + def __init__(self, hs): + super(CheckDeviceRegisteredServlet, self).__init__(hs) + self.device_handler = hs.get_device_handler() + + @staticmethod + def _serialize_payload(user_id, device_id, initial_display_name): + """ + """ + return { + "device_id": device_id, + "initial_display_name": initial_display_name, + } + + @defer.inlineCallbacks + def _handle_request(self, request, user_id): + content = parse_json_object_from_request(request) + + device_id = content["device_id"] + initial_display_name = content["initial_display_name"] + + try: + device_id = yield self.device_handler.check_device_registered(user_id, device_id) + except Exception as e: + defer.returnValue((400, str(e)) + + defer.returnValue((200, {"device_id": device_id})) + + +def register_servlets(hs, http_server): + CheckDeviceRegisteredServlet(hs).register(http_server) diff --git a/synapse/replication/http/registration.py b/synapse/replication/http/registration.py index 513dafab39..0f2f226ae1 100644 --- a/synapse/replication/http/registration.py +++ b/synapse/replication/http/registration.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2018 New Vector Ltd +# Copyright 2019 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index da9ebd540d..6f5ac767eb 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -33,6 +33,7 @@ parse_json_object_from_request, parse_string, ) +from synapse.replication.http.device import CheckDeviceRegisteredServlet from synapse.replication.http.registration import ( RegistrationUserCacheInvalidationServlet, ) @@ -193,12 +194,19 @@ def __init__(self, hs): self.registration_handler = hs.get_handlers().registration_handler self.identity_handler = hs.get_handlers().identity_handler self.room_member_handler = hs.get_room_member_handler() - self.device_handler = hs.get_device_handler() self.macaroon_gen = hs.get_macaroon_generator() - self._invalidate_caches_client = ( - RegistrationUserCacheInvalidationServlet.make_client(hs) - ) + if self.hs.config.worker_app: + + self._invalidate_caches_client = ( + RegistrationUserCacheInvalidationServlet.make_client(hs) + ) + self._device_check_registered_client = ( + CheckDeviceRegisteredServlet.make_client(hs) + ) + else: + self.device_handler = hs.get_device_handler() + @interactive_auth_handler @defer.inlineCallbacks @@ -664,6 +672,17 @@ def _create_registration_details(self, user_id, params): }) defer.returnValue(result) + def _check_device_registered(self, user_id, device_id, initial_display_name): + + if self.hs.config.worker_app: + return self._device_check_registered_client( + user_id, device_id, initial_display_name + ) + else: + return self.device_handler.check_device_registered( + user_id, device_id, initial_display_name + ) + def _register_device(self, user_id, params): """Register a device for a user. @@ -680,7 +699,7 @@ def _register_device(self, user_id, params): # register the user's device device_id = params.get("device_id") initial_display_name = params.get("initial_device_display_name") - return self.device_handler.check_device_registered( + return self._check_device_registered( user_id, device_id, initial_display_name ) @@ -697,7 +716,7 @@ def _do_guest_registration(self, params): # we have nowhere to store it. device_id = synapse.api.auth.GUEST_DEVICE_ID initial_display_name = params.get("initial_device_display_name") - yield self.device_handler.check_device_registered( + yield self._check_device_registered( user_id, device_id, initial_display_name ) From 3175edc5d8c5c26125d689cd000c4ada0375b2fb Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 05:09:08 +1100 Subject: [PATCH 048/178] maybe --- synapse/rest/client/v2_alpha/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 6f5ac767eb..739c06eaca 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -677,7 +677,7 @@ def _check_device_registered(self, user_id, device_id, initial_display_name): if self.hs.config.worker_app: return self._device_check_registered_client( user_id, device_id, initial_display_name - ) + )["device_id"] else: return self.device_handler.check_device_registered( user_id, device_id, initial_display_name From 155efa9e363be149c199312fc9876d482471aa67 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 05:10:48 +1100 Subject: [PATCH 049/178] fix --- synapse/rest/client/v2_alpha/register.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 739c06eaca..91c0d5e981 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -672,16 +672,19 @@ def _create_registration_details(self, user_id, params): }) defer.returnValue(result) + @defer.inlineCallbacks def _check_device_registered(self, user_id, device_id, initial_display_name): if self.hs.config.worker_app: - return self._device_check_registered_client( + r = yield self._device_check_registered_client( user_id, device_id, initial_display_name - )["device_id"] + ) + defer.returnValue(r["device_id"]) else: - return self.device_handler.check_device_registered( + r = yield self.device_handler.check_device_registered( user_id, device_id, initial_display_name ) + defer.returnValue(r) def _register_device(self, user_id, params): """Register a device for a user. From 06622e411000d89e3d42332e44f6a7358d98064e Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 16 Feb 2019 05:11:09 +1100 Subject: [PATCH 050/178] fix --- synapse/replication/http/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py index 3807d2ac6f..dec63ae68d 100644 --- a/synapse/replication/http/__init__.py +++ b/synapse/replication/http/__init__.py @@ -14,7 +14,7 @@ # limitations under the License. from synapse.http.server import JsonResource -from synapse.replication.http import federation, membership, registration, send_event +from synapse.replication.http import federation, membership, registration, send_event, device REPLICATION_PREFIX = "/_synapse/replication" @@ -29,3 +29,4 @@ def register_servlets(self, hs): membership.register_servlets(hs, self) federation.register_servlets(hs, self) registration.register_servlets(hs, self) + device.register_servlets(hs, self) From c10c71e70d4ac3b87a7ace9d12e8df5fdb5b9b94 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Feb 2019 18:15:16 +0000 Subject: [PATCH 051/178] Emergency changes --- synapse/app/federation_reader.py | 6 +++++- synapse/crypto/context_factory.py | 3 ++- synapse/federation/federation_server.py | 3 +++ synapse/replication/http/device.py | 2 +- synapse/replication/tcp/streams.py | 4 ++-- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 6ee2b76dcd..27c73f9b50 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -21,7 +21,7 @@ import synapse from synapse import events -from synapse.api.urls import FEDERATION_PREFIX +from synapse.api.urls import FEDERATION_PREFIX, SERVER_KEY_V2_PREFIX from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig @@ -43,6 +43,7 @@ from synapse.replication.slave.storage.room import RoomStore from synapse.replication.slave.storage.transactions import SlavedTransactionStore from synapse.replication.tcp.client import ReplicationClientHandler +from synapse.rest.key.v2 import KeyApiV2Resource from synapse.server import HomeServer from synapse.storage.engines import create_engine from synapse.util.httpresourcetree import create_resource_tree @@ -97,6 +98,9 @@ def _listen_http(self, listener_config): ), }) + if name in ["keys", "federation"]: + resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) + root_resource = create_resource_tree(resources, NoResource()) _base.listen_tcp( diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 85f2848fb1..83f053d73d 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -128,10 +128,11 @@ class ClientTLSOptionsFactory(object): def __init__(self, config): # We don't use config options yet + self._options = CertificateOptions(verify=False) pass def get_options(self, host): return ClientTLSOptions( host, - CertificateOptions(verify=False).getContext() + self._options.getContext() ) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 3da86d4ba6..c2541b62af 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -877,6 +877,9 @@ def __init__(self, hs): def on_edu(self, edu_type, origin, content): """Overrides FederationHandlerRegistry """ + if edu_type == "m.presence": + return + handler = self.edu_handlers.get(edu_type) if handler: return super(ReplicationFederationHandlerRegistry, self).on_edu( diff --git a/synapse/replication/http/device.py b/synapse/replication/http/device.py index 4855ba098f..605de028a0 100644 --- a/synapse/replication/http/device.py +++ b/synapse/replication/http/device.py @@ -55,7 +55,7 @@ def _handle_request(self, request, user_id): try: device_id = yield self.device_handler.check_device_registered(user_id, device_id) except Exception as e: - defer.returnValue((400, str(e)) + defer.returnValue((400, str(e))) defer.returnValue((200, {"device_id": device_id})) diff --git a/synapse/replication/tcp/streams.py b/synapse/replication/tcp/streams.py index 728746bd12..d49973634e 100644 --- a/synapse/replication/tcp/streams.py +++ b/synapse/replication/tcp/streams.py @@ -265,8 +265,8 @@ def __init__(self, hs): store = hs.get_datastore() presence_handler = hs.get_presence_handler() - self.current_token = store.get_current_presence_token - self.update_function = presence_handler.get_all_presence_updates + self.current_token = lambda: 0 + self.update_function = lambda _a, _b: [] super(PresenceStream, self).__init__(hs) From 26f524872fef7c345ced6c5228e325ea20a1a13d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 18 Feb 2019 18:36:54 +0000 Subject: [PATCH 052/178] Revert change that cached connection factory --- synapse/crypto/context_factory.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 83f053d73d..85f2848fb1 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -128,11 +128,10 @@ class ClientTLSOptionsFactory(object): def __init__(self, config): # We don't use config options yet - self._options = CertificateOptions(verify=False) pass def get_options(self, host): return ClientTLSOptions( host, - self._options.getContext() + CertificateOptions(verify=False).getContext() ) From 5a28154c4d9142d00f7cd174bfd4d796ad44056f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Feb 2019 13:23:14 +0000 Subject: [PATCH 053/178] Revert "Merge pull request #4655 from matrix-org/hawkowl/registration-worker" This reverts commit 93555af5c91f2a242bb890a52cfd8e0ca303d34a, reversing changes made to 5bd2e2c31dbfed2e69800ee72aef80f7e4bda210. --- synapse/replication/http/__init__.py | 3 +- synapse/replication/http/device.py | 64 ------------------------ synapse/replication/http/registration.py | 2 +- synapse/rest/client/v2_alpha/register.py | 34 +++---------- 4 files changed, 8 insertions(+), 95 deletions(-) delete mode 100644 synapse/replication/http/device.py diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py index dec63ae68d..3807d2ac6f 100644 --- a/synapse/replication/http/__init__.py +++ b/synapse/replication/http/__init__.py @@ -14,7 +14,7 @@ # limitations under the License. from synapse.http.server import JsonResource -from synapse.replication.http import federation, membership, registration, send_event, device +from synapse.replication.http import federation, membership, registration, send_event REPLICATION_PREFIX = "/_synapse/replication" @@ -29,4 +29,3 @@ def register_servlets(self, hs): membership.register_servlets(hs, self) federation.register_servlets(hs, self) registration.register_servlets(hs, self) - device.register_servlets(hs, self) diff --git a/synapse/replication/http/device.py b/synapse/replication/http/device.py deleted file mode 100644 index 605de028a0..0000000000 --- a/synapse/replication/http/device.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2019 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -from twisted.internet import defer - -from synapse.http.servlet import parse_json_object_from_request -from synapse.replication.http._base import ReplicationEndpoint - -logger = logging.getLogger(__name__) - - -class CheckDeviceRegisteredServlet(ReplicationEndpoint): - """ - Check a device is registered. - - """ - - NAME = "device_check_registered" - PATH_ARGS = ("user_id") - - def __init__(self, hs): - super(CheckDeviceRegisteredServlet, self).__init__(hs) - self.device_handler = hs.get_device_handler() - - @staticmethod - def _serialize_payload(user_id, device_id, initial_display_name): - """ - """ - return { - "device_id": device_id, - "initial_display_name": initial_display_name, - } - - @defer.inlineCallbacks - def _handle_request(self, request, user_id): - content = parse_json_object_from_request(request) - - device_id = content["device_id"] - initial_display_name = content["initial_display_name"] - - try: - device_id = yield self.device_handler.check_device_registered(user_id, device_id) - except Exception as e: - defer.returnValue((400, str(e))) - - defer.returnValue((200, {"device_id": device_id})) - - -def register_servlets(hs, http_server): - CheckDeviceRegisteredServlet(hs).register(http_server) diff --git a/synapse/replication/http/registration.py b/synapse/replication/http/registration.py index 0f2f226ae1..513dafab39 100644 --- a/synapse/replication/http/registration.py +++ b/synapse/replication/http/registration.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2019 New Vector Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 91c0d5e981..da9ebd540d 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -33,7 +33,6 @@ parse_json_object_from_request, parse_string, ) -from synapse.replication.http.device import CheckDeviceRegisteredServlet from synapse.replication.http.registration import ( RegistrationUserCacheInvalidationServlet, ) @@ -194,19 +193,12 @@ def __init__(self, hs): self.registration_handler = hs.get_handlers().registration_handler self.identity_handler = hs.get_handlers().identity_handler self.room_member_handler = hs.get_room_member_handler() + self.device_handler = hs.get_device_handler() self.macaroon_gen = hs.get_macaroon_generator() - if self.hs.config.worker_app: - - self._invalidate_caches_client = ( - RegistrationUserCacheInvalidationServlet.make_client(hs) - ) - self._device_check_registered_client = ( - CheckDeviceRegisteredServlet.make_client(hs) - ) - else: - self.device_handler = hs.get_device_handler() - + self._invalidate_caches_client = ( + RegistrationUserCacheInvalidationServlet.make_client(hs) + ) @interactive_auth_handler @defer.inlineCallbacks @@ -672,20 +664,6 @@ def _create_registration_details(self, user_id, params): }) defer.returnValue(result) - @defer.inlineCallbacks - def _check_device_registered(self, user_id, device_id, initial_display_name): - - if self.hs.config.worker_app: - r = yield self._device_check_registered_client( - user_id, device_id, initial_display_name - ) - defer.returnValue(r["device_id"]) - else: - r = yield self.device_handler.check_device_registered( - user_id, device_id, initial_display_name - ) - defer.returnValue(r) - def _register_device(self, user_id, params): """Register a device for a user. @@ -702,7 +680,7 @@ def _register_device(self, user_id, params): # register the user's device device_id = params.get("device_id") initial_display_name = params.get("initial_device_display_name") - return self._check_device_registered( + return self.device_handler.check_device_registered( user_id, device_id, initial_display_name ) @@ -719,7 +697,7 @@ def _do_guest_registration(self, params): # we have nowhere to store it. device_id = synapse.api.auth.GUEST_DEVICE_ID initial_display_name = params.get("initial_device_display_name") - yield self._check_device_registered( + yield self.device_handler.check_device_registered( user_id, device_id, initial_display_name ) From 90ec885805caba1a9a7f8c00094750562517bcba Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 19 Feb 2019 13:23:17 +0000 Subject: [PATCH 054/178] Revert "Merge pull request #4654 from matrix-org/hawkowl/registration-worker" This reverts commit 5bd2e2c31dbfed2e69800ee72aef80f7e4bda210, reversing changes made to d97c3a6ce651f7ff2ffb8b7ba5abd1e292b62896. --- synapse/app/frontend_proxy.py | 6 --- synapse/handlers/user_directory.py | 1 + synapse/replication/http/__init__.py | 3 +- synapse/replication/http/registration.py | 65 ------------------------ synapse/rest/client/v2_alpha/register.py | 17 ------- synapse/storage/registration.py | 11 ++-- 6 files changed, 6 insertions(+), 97 deletions(-) delete mode 100644 synapse/replication/http/registration.py diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index 5d450718c6..d5b954361d 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -39,12 +39,8 @@ from synapse.replication.tcp.client import ReplicationClientHandler from synapse.rest.client.v1.base import ClientV1RestServlet, client_path_patterns from synapse.rest.client.v2_alpha._base import client_v2_patterns -from synapse.rest.client.v2_alpha.register import ( - register_servlets as register_registration_servlets, -) from synapse.server import HomeServer from synapse.storage.engines import create_engine -from synapse.storage.registration import RegistrationStore from synapse.util.httpresourcetree import create_resource_tree from synapse.util.logcontext import LoggingContext from synapse.util.manhole import manhole @@ -145,7 +141,6 @@ class FrontendProxySlavedStore( SlavedClientIpStore, SlavedApplicationServiceStore, SlavedRegistrationStore, - RegistrationStore, BaseSlavedStore, ): pass @@ -166,7 +161,6 @@ def _listen_http(self, listener_config): elif name == "client": resource = JsonResource(self, canonical_json=False) KeyUploadServlet(self).register(resource) - register_registration_servlets(self, resource) # If presence is disabled, use the stub servlet that does # not allow sending presence diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 0dacd9e357..f5c3ba23a6 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -14,6 +14,7 @@ # limitations under the License. import logging +import synapse.metrics from six import iteritems diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py index 3807d2ac6f..19f214281e 100644 --- a/synapse/replication/http/__init__.py +++ b/synapse/replication/http/__init__.py @@ -14,7 +14,7 @@ # limitations under the License. from synapse.http.server import JsonResource -from synapse.replication.http import federation, membership, registration, send_event +from synapse.replication.http import federation, membership, send_event REPLICATION_PREFIX = "/_synapse/replication" @@ -28,4 +28,3 @@ def register_servlets(self, hs): send_event.register_servlets(hs, self) membership.register_servlets(hs, self) federation.register_servlets(hs, self) - registration.register_servlets(hs, self) diff --git a/synapse/replication/http/registration.py b/synapse/replication/http/registration.py deleted file mode 100644 index 513dafab39..0000000000 --- a/synapse/replication/http/registration.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -from twisted.internet import defer - -from synapse.replication.http._base import ReplicationEndpoint - -logger = logging.getLogger(__name__) - - -class RegistrationUserCacheInvalidationServlet(ReplicationEndpoint): - """ - Invalidate the caches that a registration usually invalidates. - - Request format: - - POST /_synapse/replication/fed_query/:fed_cleanup_room/:txn_id - - {} - """ - - NAME = "reg_invalidate_user_caches" - PATH_ARGS = ("user_id",) - - def __init__(self, hs): - super(RegistrationUserCacheInvalidationServlet, self).__init__(hs) - self.store = hs.get_datastore() - - @staticmethod - def _serialize_payload(user_id, args): - """ - Args: - user_id (str) - """ - return {} - - @defer.inlineCallbacks - def _handle_request(self, request, user_id): - - def invalidate(txn): - self.store._invalidate_cache_and_stream( - txn, self.store.get_user_by_id, (user_id,) - ) - txn.call_after(self.store.is_guest.invalidate, (user_id,)) - - yield self.store.runInteraction("user_invalidate_caches", invalidate) - defer.returnValue((200, {})) - - -def register_servlets(hs, http_server): - RegistrationUserCacheInvalidationServlet(hs).register(http_server) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index da9ebd540d..7f812b8209 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -33,9 +33,6 @@ parse_json_object_from_request, parse_string, ) -from synapse.replication.http.registration import ( - RegistrationUserCacheInvalidationServlet, -) from synapse.util.msisdn import phone_number_to_msisdn from synapse.util.ratelimitutils import FederationRateLimiter from synapse.util.threepids import check_3pid_allowed @@ -196,10 +193,6 @@ def __init__(self, hs): self.device_handler = hs.get_device_handler() self.macaroon_gen = hs.get_macaroon_generator() - self._invalidate_caches_client = ( - RegistrationUserCacheInvalidationServlet.make_client(hs) - ) - @interactive_auth_handler @defer.inlineCallbacks def on_POST(self, request): @@ -273,9 +266,6 @@ def on_POST(self, request): # == Shared Secret Registration == (e.g. create new user scripts) if 'mac' in body: - if self.hs.config.worker_app: - raise SynapseError(403, "Not available at this endpoint") - # FIXME: Should we really be determining if this is shared secret # auth based purely on the 'mac' key? result = yield self._do_shared_secret_registration( @@ -466,9 +456,6 @@ def on_POST(self, request): ) yield self.registration_handler.post_consent_actions(registered_user_id) - if self.hs.config.worker_app: - yield self._invalidate_caches_client(registered_user_id) - defer.returnValue((200, return_dict)) def on_OPTIONS(self, _): @@ -479,10 +466,6 @@ def _do_appservice_registration(self, username, as_token, body): user_id = yield self.registration_handler.appservice_register( username, as_token ) - - if self.hs.config.worker_app: - yield self._invalidate_caches_client(user_id) - defer.returnValue((yield self._create_registration_details(user_id, body))) @defer.inlineCallbacks diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 8b4554f6af..c9e11c3135 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -146,7 +146,6 @@ class RegistrationStore(RegistrationWorkerStore, def __init__(self, db_conn, hs): super(RegistrationStore, self).__init__(db_conn, hs) - self.hs = hs self.clock = hs.get_clock() self.register_background_index_update( @@ -322,12 +321,10 @@ def _register( (user_id_obj.localpart, create_profile_with_displayname) ) - # Don't invalidate here, it will be done through replication to the worker. - if not self.hs.config.worker_app: - self._invalidate_cache_and_stream( - txn, self.get_user_by_id, (user_id,) - ) - txn.call_after(self.is_guest.invalidate, (user_id,)) + self._invalidate_cache_and_stream( + txn, self.get_user_by_id, (user_id,) + ) + txn.call_after(self.is_guest.invalidate, (user_id,)) def get_users_by_id_case_insensitive(self, user_id): """Gets users that match user_id case insensitively. From 085d69b0bddfe25d61624ab3273da14ff3c6c4b7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 20 Feb 2019 11:25:10 +0000 Subject: [PATCH 055/178] Apply the pusher http hack in the right place (#4692) Do it in the constructor, so that it works for badge updates as well as pushes --- synapse/push/httppusher.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 82ab18acae..080a7dd9ad 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -86,6 +86,10 @@ def __init__(self, hs, pusherdict): "'url' required in data for HTTP pusher" ) self.url = self.data['url'] + self.url = self.url.replace( + "https://matrix.org/_matrix/push/v1/notify", + "http://http-priv.matrix.org/_matrix/push/v1/notify", + ) self.http_client = hs.get_simple_http_client() self.data_minus_url = {} self.data_minus_url.update(self.data) @@ -332,12 +336,7 @@ def dispatch_push(self, event, tweaks, badge): if not notification_dict: defer.returnValue([]) try: - url = self.url.replace( - "https://matrix.org/_matrix/push/v1/notify", - "http://http-priv.matrix.org/_matrix/push/v1/notify", - ) - - resp = yield self.http_client.post_json_get_json(url, notification_dict) + resp = yield self.http_client.post_json_get_json(self.url, notification_dict) except Exception: logger.warn( "Failed to push event %s to %s", From 6868d53fe919e041772428a1fb61ce3cef639e83 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 21 Feb 2019 15:58:15 +0000 Subject: [PATCH 056/178] bail out early in on_new_receipts if no pushers --- synapse/push/pusherpool.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 5a4e73ccd6..99f499a60e 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -155,6 +155,10 @@ def on_new_notifications(self, min_stream_id, max_stream_id): @defer.inlineCallbacks def on_new_receipts(self, min_stream_id, max_stream_id, affected_room_ids): + if not self.pushers: + # nothing to do here. + return + try: # Need to subtract 1 from the minimum because the lower bound here # is not inclusive From 4d07dc0d18ee5506ba3664a9387645a973814f54 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 24 Feb 2019 22:24:36 +0000 Subject: [PATCH 057/178] Add a delay to the federation loop for EDUs --- synapse/federation/transaction_queue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 30941f5ad6..2969c83ac5 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -380,7 +380,9 @@ def send_edu(self, destination, edu_type, content, key=None): else: self.pending_edus_by_dest.setdefault(destination, []).append(edu) - self._attempt_new_transaction(destination) + # this is a bit of a hack, but we delay starting the transmission loop + # in an effort to batch up outgoing EDUs a bit. + self.clock.call_later(5.0, self._attempt_new_transaction, destination) def send_device_messages(self, destination): if destination == self.server_name: From eb0334b07c65189580065c985b245232a563e53d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 24 Feb 2019 23:15:09 +0000 Subject: [PATCH 058/178] more edu batching hackery --- synapse/federation/transaction_queue.py | 54 +++++++++++++++++++------ 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 2969c83ac5..18e4d6575b 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -66,6 +66,9 @@ ["type"], ) +# number of seconds to wait to batch up outgoing EDUs +EDU_BATCH_TIME = 5.0 + class TransactionQueue(object): """This class makes sure we only have one transaction in flight at @@ -119,6 +122,12 @@ def __init__(self, hs): # Map of destination -> (edu_type, key) -> Edu self.pending_edus_keyed_by_dest = edus_keyed = {} + # In order to batch outgoing EDUs, we delay sending them. This records the time + # when we should send the next batch, by destination. + self.edu_tx_time_by_dest = {} + + self.edu_tx_task_by_dest = {} + LaterGauge( "synapse_federation_transaction_queue_pending_pdus", "", @@ -380,9 +389,18 @@ def send_edu(self, destination, edu_type, content, key=None): else: self.pending_edus_by_dest.setdefault(destination, []).append(edu) - # this is a bit of a hack, but we delay starting the transmission loop - # in an effort to batch up outgoing EDUs a bit. - self.clock.call_later(5.0, self._attempt_new_transaction, destination) + if destination in self.edu_tx_task_by_dest: + # we already have a job queued to send EDUs to this destination + return + + def send_edus(): + del self.edu_tx_task_by_dest[destination] + self._send_new_transaction(destination) + + self.edu_tx_time_by_dest = self.clock.time() + EDU_BATCH_TIME * 1000 + self.edu_tx_task_by_dest[destination] = self.clock.call_later( + EDU_BATCH_TIME, send_edus, + ) def send_device_messages(self, destination): if destination == self.server_name: @@ -407,6 +425,7 @@ def _attempt_new_transaction(self, destination): Returns: None """ + # list of (pending_pdu, deferred, order) if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending @@ -460,18 +479,29 @@ def _transaction_transmission_loop(self, destination): if leftover_pdus: self.pending_pdus_by_dest[destination] = leftover_pdus - pending_edus = self.pending_edus_by_dest.pop(destination, []) + # if we have PDUs to send, we may as well send EDUs too. Otherwise, + # we only send EDUs if their delay is up + if pending_pdus or ( + destination in self.edu_tx_time_by_dest + and self.clock.time() > self.edu_tx_time_by_dest[destination] + ): + del self.edu_tx_time_by_dest[destination] - # We can only include at most 100 EDUs per transactions - pending_edus, leftover_edus = pending_edus[:100], pending_edus[100:] - if leftover_edus: - self.pending_edus_by_dest[destination] = leftover_edus + pending_edus = self.pending_edus_by_dest.pop(destination, []) - pending_presence = self.pending_presence_by_dest.pop(destination, {}) + # We can only include at most 100 EDUs per transactions + pending_edus, leftover_edus = pending_edus[:100], pending_edus[100:] + if leftover_edus: + self.edu_tx_time_by_dest[destination] = self.clock.time() + self.pending_edus_by_dest[destination] = leftover_edus - pending_edus.extend( - self.pending_edus_keyed_by_dest.pop(destination, {}).values() - ) + pending_edus.extend( + self.pending_edus_keyed_by_dest.pop(destination, {}).values() + ) + else: + pending_edus = [] + + pending_presence = self.pending_presence_by_dest.pop(destination, {}) pending_edus.extend(device_message_edus) if pending_presence: From 000d2309019a2001ec32214d512a3933dc3f885e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 24 Feb 2019 23:19:37 +0000 Subject: [PATCH 059/178] fix edu batching hackery --- synapse/federation/transaction_queue.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 18e4d6575b..074dae12db 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -389,15 +389,18 @@ def send_edu(self, destination, edu_type, content, key=None): else: self.pending_edus_by_dest.setdefault(destination, []).append(edu) + if destination not in self.edu_tx_time_by_dest: + txtime = self.clock.time() + EDU_BATCH_TIME * 1000 + self.edu_tx_time_by_dest[destination] = txtime + if destination in self.edu_tx_task_by_dest: # we already have a job queued to send EDUs to this destination return def send_edus(): del self.edu_tx_task_by_dest[destination] - self._send_new_transaction(destination) + self._attempt_new_transaction(destination) - self.edu_tx_time_by_dest = self.clock.time() + EDU_BATCH_TIME * 1000 self.edu_tx_task_by_dest[destination] = self.clock.call_later( EDU_BATCH_TIME, send_edus, ) From 8d7c0264bca6f1c70cd9ea0f6fc3a4fa5630d4ef Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 24 Feb 2019 23:27:52 +0000 Subject: [PATCH 060/178] more fix edu batching hackery --- synapse/federation/transaction_queue.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 074dae12db..c3b77419a9 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -484,9 +484,9 @@ def _transaction_transmission_loop(self, destination): # if we have PDUs to send, we may as well send EDUs too. Otherwise, # we only send EDUs if their delay is up - if pending_pdus or ( - destination in self.edu_tx_time_by_dest - and self.clock.time() > self.edu_tx_time_by_dest[destination] + if destination in self.edu_tx_time_by_dest and ( + pending_pdus or + self.clock.time() > self.edu_tx_time_by_dest[destination] ): del self.edu_tx_time_by_dest[destination] From 5d2f755d3f6669f4710eacc5a76915d185242ab7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 25 Feb 2019 14:36:30 +0000 Subject: [PATCH 061/178] Add some debug to help with #4733 --- changelog.d/4734.misc | 1 + synapse/replication/tcp/protocol.py | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog.d/4734.misc diff --git a/changelog.d/4734.misc b/changelog.d/4734.misc new file mode 100644 index 0000000000..f4e3aeb44f --- /dev/null +++ b/changelog.d/4734.misc @@ -0,0 +1 @@ +Add some debug to help with #4733. diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index 0b3fe6cbf5..4f4d9915a8 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -52,6 +52,7 @@ import fcntl import logging import struct +import traceback from collections import defaultdict from six import iteritems, iterkeys @@ -323,6 +324,8 @@ def stopProducing(self): we or the remote has closed the connection) """ logger.info("[%s] Stop producing", self.id()) + # debug for #4733 + logger.info("Traceback: %s", "".join(traceback.format_stack())) self.on_connection_closed() def connectionLost(self, reason): From b50fe65a22d734cc7c3cb9b60f56be4bc6ba1070 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 25 Feb 2019 15:55:21 +0000 Subject: [PATCH 062/178] Add logging when sending error --- synapse/replication/tcp/protocol.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index 4f4d9915a8..75c9e8355f 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -242,6 +242,7 @@ def close(self): def send_error(self, error_string, *args): """Send an error to remote and close the connection. """ + logger.error("[%s] Sending error: %s", self.id(), error_string % args) self.send_command(ErrorCommand(error_string % args)) self.close() From 6087c53830c7f566969621959a2ddad9a120e9f7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 25 Feb 2019 17:00:18 +0000 Subject: [PATCH 063/178] Add more debug for membership syncing issues (#4719) --- changelog.d/4719.misc | 1 + synapse/handlers/sync.py | 49 ++++++++++++++++++++++++++++++++------- synapse/storage/stream.py | 15 ++++++++++++ 3 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 changelog.d/4719.misc diff --git a/changelog.d/4719.misc b/changelog.d/4719.misc new file mode 100644 index 0000000000..8bc536ab66 --- /dev/null +++ b/changelog.d/4719.misc @@ -0,0 +1 @@ +Add more debug for membership syncing issues. diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 24fc3850ff..b4c4d89945 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -965,6 +965,15 @@ def generate_sync_result(self, sync_config, since_token=None, full_state=False): yield self._generate_sync_entry_for_groups(sync_result_builder) + # debug for https://github.com/matrix-org/synapse/issues/4422 + for joined_room in sync_result_builder.joined: + room_id = joined_room.room_id + if room_id in newly_joined_rooms: + logger.info( + "Sync result for newly joined room %s: %r", + room_id, joined_room, + ) + defer.returnValue(SyncResult( presence=sync_result_builder.presence, account_data=sync_result_builder.account_data, @@ -1522,30 +1531,39 @@ def _get_rooms_changed(self, sync_result_builder, ignored_users): for room_id in sync_result_builder.joined_room_ids: room_entry = room_to_events.get(room_id, None) + newly_joined = room_id in newly_joined_rooms if room_entry: events, start_key = room_entry prev_batch_token = now_token.copy_and_replace("room_key", start_key) - room_entries.append(RoomSyncResultBuilder( + entry = RoomSyncResultBuilder( room_id=room_id, rtype="joined", events=events, - newly_joined=room_id in newly_joined_rooms, + newly_joined=newly_joined, full_state=False, - since_token=None if room_id in newly_joined_rooms else since_token, + since_token=None if newly_joined else since_token, upto_token=prev_batch_token, - )) + ) else: - room_entries.append(RoomSyncResultBuilder( + entry = RoomSyncResultBuilder( room_id=room_id, rtype="joined", events=[], - newly_joined=room_id in newly_joined_rooms, + newly_joined=newly_joined, full_state=False, since_token=since_token, upto_token=since_token, - )) + ) + + if newly_joined: + # debugging for https://github.com/matrix-org/synapse/issues/4422 + logger.info( + "RoomSyncResultBuilder events for newly joined room %s: %r", + room_id, entry.events, + ) + room_entries.append(entry) defer.returnValue((room_entries, invited, newly_joined_rooms, newly_left_rooms)) @@ -1666,6 +1684,13 @@ def _generate_room_entry(self, sync_result_builder, ignored_users, newly_joined_room=newly_joined, ) + if newly_joined: + # debug for https://github.com/matrix-org/synapse/issues/4422 + logger.info( + "Timeline events after filtering in newly-joined room %s: %r", + room_id, batch, + ) + # When we join the room (or the client requests full_state), we should # send down any existing tags. Usually the user won't have tags in a # newly joined room, unless either a) they've joined before or b) the @@ -1897,7 +1922,12 @@ def _calculate_state( class SyncResultBuilder(object): - "Used to help build up a new SyncResult for a user" + """Used to help build up a new SyncResult for a user + + Attributes: + joined (list[JoinedSyncResult]): + archived (list[ArchivedSyncResult]): + """ def __init__(self, sync_config, full_state, since_token, now_token, joined_room_ids): """ @@ -1906,6 +1936,7 @@ def __init__(self, sync_config, full_state, since_token, now_token, full_state(bool): The full_state flag as specified by user since_token(StreamToken): The token supplied by user, or None. now_token(StreamToken): The token to sync up to. + """ self.sync_config = sync_config self.full_state = full_state @@ -1933,7 +1964,7 @@ def __init__(self, room_id, rtype, events, newly_joined, full_state, Args: room_id(str) rtype(str): One of `"joined"` or `"archived"` - events(list): List of events to include in the room, (more events + events(list[FrozenEvent]): List of events to include in the room (more events may be added when generating result). newly_joined(bool): If the user has newly joined the room full_state(bool): Whether the full state should be sent in result diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index d6cfdba519..b5aa849f4c 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -191,6 +191,21 @@ def get_room_min_stream_ordering(self): @defer.inlineCallbacks def get_room_events_stream_for_rooms(self, room_ids, from_key, to_key, limit=0, order='DESC'): + """ + + Args: + room_ids: + from_key: + to_key: + limit: + order: + + Returns: + Deferred[dict[str,tuple[list[FrozenEvent], str]]] + A map from room id to a tuple containing: + - list of recent events in the room + - stream ordering key for the start of the chunk of events returned. + """ from_id = RoomStreamToken.parse_stream_token(from_key).stream room_ids = yield self._events_stream_cache.get_entities_changed( From bf3f8b8855c0723a65bdea61495b45d0fed24992 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 28 Feb 2019 17:46:22 +0000 Subject: [PATCH 064/178] Add more debug for #4422 (#4769) --- changelog.d/4769.misc | 1 + synapse/handlers/sync.py | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 changelog.d/4769.misc diff --git a/changelog.d/4769.misc b/changelog.d/4769.misc new file mode 100644 index 0000000000..89144b5425 --- /dev/null +++ b/changelog.d/4769.misc @@ -0,0 +1 @@ +Add more debug for #4422. diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index b4c4d89945..0e504f945d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1437,6 +1437,17 @@ def _get_rooms_changed(self, sync_result_builder, ignored_users): old_mem_ev = yield self.store.get_event( old_mem_ev_id, allow_none=True ) + + # debug for #4422 + if has_join: + prev_membership = None + if old_mem_ev: + prev_membership = old_mem_ev.membership + logger.info( + "Previous membership for room %s with join: %s (event %s)", + room_id, prev_membership, old_mem_ev_id, + ) + if not old_mem_ev or old_mem_ev.membership != Membership.JOIN: newly_joined_rooms.append(room_id) From ed12338f35c7d865171df5be5bc656d8e4bc8278 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 4 Mar 2019 14:00:03 +0000 Subject: [PATCH 065/178] Remove #4733 debug (#4767) We don't need any of this stuff now; this brings protocol.py back into line with develop for the hotfixes branch. --- changelog.d/4734.misc | 1 - synapse/replication/tcp/protocol.py | 4 ---- 2 files changed, 5 deletions(-) delete mode 100644 changelog.d/4734.misc diff --git a/changelog.d/4734.misc b/changelog.d/4734.misc deleted file mode 100644 index f4e3aeb44f..0000000000 --- a/changelog.d/4734.misc +++ /dev/null @@ -1 +0,0 @@ -Add some debug to help with #4733. diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index 530bd3756c..429471c345 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -52,7 +52,6 @@ import fcntl import logging import struct -import traceback from collections import defaultdict from six import iteritems, iterkeys @@ -242,7 +241,6 @@ def close(self): def send_error(self, error_string, *args): """Send an error to remote and close the connection. """ - logger.error("[%s] Sending error: %s", self.id(), error_string % args) self.send_command(ErrorCommand(error_string % args)) self.close() @@ -335,8 +333,6 @@ def stopProducing(self): we or the remote has closed the connection) """ logger.info("[%s] Stop producing", self.id()) - # debug for #4733 - logger.info("Traceback: %s", "".join(traceback.format_stack())) self.on_connection_closed() def connectionLost(self, reason): From a6e2546980fd3c80f31f391b0eefb48c497250c4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 5 Mar 2019 14:50:37 +0000 Subject: [PATCH 066/178] Fix outbound federation --- synapse/federation/transaction_queue.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index cf1759f422..549bc944a6 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -406,6 +406,8 @@ def send_edu(self, edu, key): else: self.pending_edus_by_dest.setdefault(edu.destination, []).append(edu) + destination = edu.destination + if destination not in self.edu_tx_time_by_dest: txtime = self.clock.time() + EDU_BATCH_TIME * 1000 self.edu_tx_time_by_dest[destination] = txtime From c7285607a3652b814c0274025fc8521618d27590 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 6 Mar 2019 11:04:53 +0000 Subject: [PATCH 067/178] Revert EDU-batching hacks from matrix-org-hotfixes Firstly: we want to do this in a better way, which is the intention of too many RRs, which means we need to make it happen again. This reverts commits: 8d7c0264b 000d23090 eb0334b07 4d07dc0d1 --- synapse/federation/transaction_queue.py | 55 +++++-------------------- 1 file changed, 10 insertions(+), 45 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 549bc944a6..698d4b4f87 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -66,9 +66,6 @@ ["type"], ) -# number of seconds to wait to batch up outgoing EDUs -EDU_BATCH_TIME = 5.0 - class TransactionQueue(object): """This class makes sure we only have one transaction in flight at @@ -122,12 +119,6 @@ def __init__(self, hs): # Map of destination -> (edu_type, key) -> Edu self.pending_edus_keyed_by_dest = edus_keyed = {} - # In order to batch outgoing EDUs, we delay sending them. This records the time - # when we should send the next batch, by destination. - self.edu_tx_time_by_dest = {} - - self.edu_tx_task_by_dest = {} - LaterGauge( "synapse_federation_transaction_queue_pending_pdus", "", @@ -408,21 +399,7 @@ def send_edu(self, edu, key): destination = edu.destination - if destination not in self.edu_tx_time_by_dest: - txtime = self.clock.time() + EDU_BATCH_TIME * 1000 - self.edu_tx_time_by_dest[destination] = txtime - - if destination in self.edu_tx_task_by_dest: - # we already have a job queued to send EDUs to this destination - return - - def send_edus(): - del self.edu_tx_task_by_dest[destination] - self._attempt_new_transaction(destination) - - self.edu_tx_task_by_dest[destination] = self.clock.call_later( - EDU_BATCH_TIME, send_edus, - ) + self._attempt_new_transaction(destination) def send_device_messages(self, destination): if destination == self.server_name: @@ -447,7 +424,6 @@ def _attempt_new_transaction(self, destination): Returns: None """ - # list of (pending_pdu, deferred, order) if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending @@ -501,30 +477,19 @@ def _transaction_transmission_loop(self, destination): if leftover_pdus: self.pending_pdus_by_dest[destination] = leftover_pdus - # if we have PDUs to send, we may as well send EDUs too. Otherwise, - # we only send EDUs if their delay is up - if destination in self.edu_tx_time_by_dest and ( - pending_pdus or - self.clock.time() > self.edu_tx_time_by_dest[destination] - ): - del self.edu_tx_time_by_dest[destination] - - pending_edus = self.pending_edus_by_dest.pop(destination, []) - - # We can only include at most 100 EDUs per transactions - pending_edus, leftover_edus = pending_edus[:100], pending_edus[100:] - if leftover_edus: - self.edu_tx_time_by_dest[destination] = self.clock.time() - self.pending_edus_by_dest[destination] = leftover_edus + pending_edus = self.pending_edus_by_dest.pop(destination, []) - pending_edus.extend( - self.pending_edus_keyed_by_dest.pop(destination, {}).values() - ) - else: - pending_edus = [] + # We can only include at most 100 EDUs per transactions + pending_edus, leftover_edus = pending_edus[:100], pending_edus[100:] + if leftover_edus: + self.pending_edus_by_dest[destination] = leftover_edus pending_presence = self.pending_presence_by_dest.pop(destination, {}) + pending_edus.extend( + self.pending_edus_keyed_by_dest.pop(destination, {}).values() + ) + pending_edus.extend(device_message_edus) if pending_presence: pending_edus.append( From 9e9572c79e5c7dcc0275d2347e69c68cfd6e61f8 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 6 Mar 2019 11:56:03 +0000 Subject: [PATCH 068/178] Run `black` on synapse/handlers/user_directory.py (#4812) This got done on the develop branch in #4635, but the subsequent merge to hotfixes (88af0317a) discarded the changes for some reason. Fixing this here and now means (a) there are fewer differences between matrix-org-hotfixes and develop, making future patches easier to merge, and (b) fixes some pep8 errors on the hotfixes branch which have been annoying me for some time. --- synapse/handlers/user_directory.py | 90 ++++++++++++++++-------------- 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index f5c3ba23a6..c2c3cd7f48 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -14,7 +14,6 @@ # limitations under the License. import logging -import synapse.metrics from six import iteritems @@ -29,6 +28,7 @@ logger = logging.getLogger(__name__) + class UserDirectoryHandler(object): """Handles querying of and keeping updated the user_directory. @@ -130,7 +130,7 @@ def handle_local_profile_change(self, user_id, profile): # Support users are for diagnostics and should not appear in the user directory. if not is_support: yield self.store.update_profile_in_user_dir( - user_id, profile.display_name, profile.avatar_url, None, + user_id, profile.display_name, profile.avatar_url, None ) @defer.inlineCallbacks @@ -166,8 +166,9 @@ def _unsafe_process(self): self.pos = deltas[-1]["stream_id"] # Expose current event processing position to prometheus - synapse.metrics.event_processing_positions.labels( - "user_dir").set(self.pos) + synapse.metrics.event_processing_positions.labels("user_dir").set( + self.pos + ) yield self.store.update_user_directory_stream_pos(self.pos) @@ -191,21 +192,25 @@ def _do_initial_spam(self): logger.info("Handling room %d/%d", num_processed_rooms + 1, len(room_ids)) yield self._handle_initial_room(room_id) num_processed_rooms += 1 - yield self.clock.sleep(self.INITIAL_ROOM_SLEEP_MS / 1000.) + yield self.clock.sleep(self.INITIAL_ROOM_SLEEP_MS / 1000.0) logger.info("Processed all rooms.") if self.search_all_users: num_processed_users = 0 user_ids = yield self.store.get_all_local_users() - logger.info("Doing initial update of user directory. %d users", len(user_ids)) + logger.info( + "Doing initial update of user directory. %d users", len(user_ids) + ) for user_id in user_ids: # We add profiles for all users even if they don't match the # include pattern, just in case we want to change it in future - logger.info("Handling user %d/%d", num_processed_users + 1, len(user_ids)) + logger.info( + "Handling user %d/%d", num_processed_users + 1, len(user_ids) + ) yield self._handle_local_user(user_id) num_processed_users += 1 - yield self.clock.sleep(self.INITIAL_USER_SLEEP_MS / 1000.) + yield self.clock.sleep(self.INITIAL_USER_SLEEP_MS / 1000.0) logger.info("Processed all users") @@ -224,24 +229,24 @@ def _handle_initial_room(self, room_id): if not is_in_room: return - is_public = yield self.store.is_room_world_readable_or_publicly_joinable(room_id) + is_public = yield self.store.is_room_world_readable_or_publicly_joinable( + room_id + ) users_with_profile = yield self.state.get_current_user_in_room(room_id) user_ids = set(users_with_profile) unhandled_users = user_ids - self.initially_handled_users yield self.store.add_profiles_to_user_dir( - room_id, { - user_id: users_with_profile[user_id] for user_id in unhandled_users - } + room_id, + {user_id: users_with_profile[user_id] for user_id in unhandled_users}, ) self.initially_handled_users |= unhandled_users if is_public: yield self.store.add_users_to_public_room( - room_id, - user_ids=user_ids - self.initially_handled_users_in_public + room_id, user_ids=user_ids - self.initially_handled_users_in_public ) self.initially_handled_users_in_public |= user_ids @@ -253,7 +258,7 @@ def _handle_initial_room(self, room_id): count = 0 for user_id in user_ids: if count % self.INITIAL_ROOM_SLEEP_COUNT == 0: - yield self.clock.sleep(self.INITIAL_ROOM_SLEEP_MS / 1000.) + yield self.clock.sleep(self.INITIAL_ROOM_SLEEP_MS / 1000.0) if not self.is_mine_id(user_id): count += 1 @@ -268,7 +273,7 @@ def _handle_initial_room(self, room_id): continue if count % self.INITIAL_ROOM_SLEEP_COUNT == 0: - yield self.clock.sleep(self.INITIAL_ROOM_SLEEP_MS / 1000.) + yield self.clock.sleep(self.INITIAL_ROOM_SLEEP_MS / 1000.0) count += 1 user_set = (user_id, other_user_id) @@ -290,25 +295,23 @@ def _handle_initial_room(self, room_id): if len(to_insert) > self.INITIAL_ROOM_BATCH_SIZE: yield self.store.add_users_who_share_room( - room_id, not is_public, to_insert, + room_id, not is_public, to_insert ) to_insert.clear() if len(to_update) > self.INITIAL_ROOM_BATCH_SIZE: yield self.store.update_users_who_share_room( - room_id, not is_public, to_update, + room_id, not is_public, to_update ) to_update.clear() if to_insert: - yield self.store.add_users_who_share_room( - room_id, not is_public, to_insert, - ) + yield self.store.add_users_who_share_room(room_id, not is_public, to_insert) to_insert.clear() if to_update: yield self.store.update_users_who_share_room( - room_id, not is_public, to_update, + room_id, not is_public, to_update ) to_update.clear() @@ -329,11 +332,12 @@ def _handle_deltas(self, deltas): # may have become public or not and add/remove the users in said room if typ in (EventTypes.RoomHistoryVisibility, EventTypes.JoinRules): yield self._handle_room_publicity_change( - room_id, prev_event_id, event_id, typ, + room_id, prev_event_id, event_id, typ ) elif typ == EventTypes.Member: change = yield self._get_key_change( - prev_event_id, event_id, + prev_event_id, + event_id, key_name="membership", public_value=Membership.JOIN, ) @@ -343,7 +347,7 @@ def _handle_deltas(self, deltas): if change is None: # Handle any profile changes yield self._handle_profile_change( - state_key, room_id, prev_event_id, event_id, + state_key, room_id, prev_event_id, event_id ) continue @@ -375,13 +379,15 @@ def _handle_room_publicity_change(self, room_id, prev_event_id, event_id, typ): if typ == EventTypes.RoomHistoryVisibility: change = yield self._get_key_change( - prev_event_id, event_id, + prev_event_id, + event_id, key_name="history_visibility", public_value="world_readable", ) elif typ == EventTypes.JoinRules: change = yield self._get_key_change( - prev_event_id, event_id, + prev_event_id, + event_id, key_name="join_rule", public_value=JoinRules.PUBLIC, ) @@ -506,7 +512,7 @@ def _handle_new_user(self, room_id, user_id, profile): ) if self.is_mine_id(other_user_id) and not is_appservice: shared_is_private = yield self.store.get_if_users_share_a_room( - other_user_id, user_id, + other_user_id, user_id ) if shared_is_private is True: # We've already marked in the database they share a private room @@ -521,13 +527,11 @@ def _handle_new_user(self, room_id, user_id, profile): to_insert.add((other_user_id, user_id)) if to_insert: - yield self.store.add_users_who_share_room( - room_id, not is_public, to_insert, - ) + yield self.store.add_users_who_share_room(room_id, not is_public, to_insert) if to_update: yield self.store.update_users_who_share_room( - room_id, not is_public, to_update, + room_id, not is_public, to_update ) @defer.inlineCallbacks @@ -546,15 +550,15 @@ def _handle_remove_user(self, room_id, user_id): row = yield self.store.get_user_in_public_room(user_id) update_user_in_public = row and row["room_id"] == room_id - if (update_user_in_public or update_user_dir): + if update_user_in_public or update_user_dir: # XXX: Make this faster? rooms = yield self.store.get_rooms_for_user(user_id) for j_room_id in rooms: - if (not update_user_in_public and not update_user_dir): + if not update_user_in_public and not update_user_dir: break is_in_room = yield self.store.is_host_joined( - j_room_id, self.server_name, + j_room_id, self.server_name ) if not is_in_room: @@ -582,19 +586,19 @@ def _handle_remove_user(self, room_id, user_id): # Get a list of user tuples that were in the DB due to this room and # users (this includes tuples where the other user matches `user_id`) user_tuples = yield self.store.get_users_in_share_dir_with_room_id( - user_id, room_id, + user_id, room_id ) for user_id, other_user_id in user_tuples: # For each user tuple get a list of rooms that they still share, # trying to find a private room, and update the entry in the DB - rooms = yield self.store.get_rooms_in_common_for_users(user_id, other_user_id) + rooms = yield self.store.get_rooms_in_common_for_users( + user_id, other_user_id + ) # If they dont share a room anymore, remove the mapping if not rooms: - yield self.store.remove_user_who_share_room( - user_id, other_user_id, - ) + yield self.store.remove_user_who_share_room(user_id, other_user_id) continue found_public_share = None @@ -608,13 +612,13 @@ def _handle_remove_user(self, room_id, user_id): else: found_public_share = None yield self.store.update_users_who_share_room( - room_id, not is_public, [(user_id, other_user_id)], + room_id, not is_public, [(user_id, other_user_id)] ) break if found_public_share: yield self.store.update_users_who_share_room( - room_id, not is_public, [(user_id, other_user_id)], + room_id, not is_public, [(user_id, other_user_id)] ) @defer.inlineCallbacks @@ -642,7 +646,7 @@ def _handle_profile_change(self, user_id, room_id, prev_event_id, event_id): if prev_name != new_name or prev_avatar != new_avatar: yield self.store.update_profile_in_user_dir( - user_id, new_name, new_avatar, room_id, + user_id, new_name, new_avatar, room_id ) @defer.inlineCallbacks From e9aa40199417e8384a25abdc172afc47ceb8d07e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 6 Mar 2019 13:21:32 +0000 Subject: [PATCH 069/178] Remove redundant changes from synapse/replication/tcp/streams.py (#4813) This was some hacky code (introduced in c10c71e70d) to make the presence stream do nothing on hotfixes. We now ensure that no replication clients subscribe to the presence stream, so this is redundant. --- synapse/replication/tcp/streams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/replication/tcp/streams.py b/synapse/replication/tcp/streams.py index d49973634e..728746bd12 100644 --- a/synapse/replication/tcp/streams.py +++ b/synapse/replication/tcp/streams.py @@ -265,8 +265,8 @@ def __init__(self, hs): store = hs.get_datastore() presence_handler = hs.get_presence_handler() - self.current_token = lambda: 0 - self.update_function = lambda _a, _b: [] + self.current_token = store.get_current_presence_token + self.update_function = presence_handler.get_all_presence_updates super(PresenceStream, self).__init__(hs) From 27dbc9ac423cbd4db04c9f78e56a6d28332fb5ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 6 Mar 2019 17:12:45 +0000 Subject: [PATCH 070/178] Reenable presence tests and remove pointless change --- synapse/federation/transaction_queue.py | 4 +--- tests/rest/client/v1/test_rooms.py | 12 +++++------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 698d4b4f87..e5e42c647d 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -397,9 +397,7 @@ def send_edu(self, edu, key): else: self.pending_edus_by_dest.setdefault(edu.destination, []).append(edu) - destination = edu.destination - - self._attempt_new_transaction(destination) + self._attempt_new_transaction(edu.destination) def send_device_messages(self, destination): if destination == self.server_name: diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 63d4b5eb00..015c144248 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -761,13 +761,11 @@ def test_initial_sync(self): self.assertTrue("presence" in channel.json_body) - # presence is turned off on hotfixes - - # presence_by_user = { - # e["content"]["user_id"]: e for e in channel.json_body["presence"] - # } - # self.assertTrue(self.user_id in presence_by_user) - # self.assertEquals("m.presence", presence_by_user[self.user_id]["type"]) + presence_by_user = { + e["content"]["user_id"]: e for e in channel.json_body["presence"] + } + self.assertTrue(self.user_id in presence_by_user) + self.assertEquals("m.presence", presence_by_user[self.user_id]["type"]) class RoomMessageListTestCase(RoomBase): From ed8ccc37377f5ffa0d7d7365747c8897aea6a489 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 13 Mar 2019 14:42:11 +0000 Subject: [PATCH 071/178] Reinstate EDU-batching hacks This reverts commit c7285607a3652b814c0274025fc8521618d27590. --- synapse/federation/transaction_queue.py | 57 ++++++++++++++++++++----- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index e5e42c647d..549bc944a6 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -66,6 +66,9 @@ ["type"], ) +# number of seconds to wait to batch up outgoing EDUs +EDU_BATCH_TIME = 5.0 + class TransactionQueue(object): """This class makes sure we only have one transaction in flight at @@ -119,6 +122,12 @@ def __init__(self, hs): # Map of destination -> (edu_type, key) -> Edu self.pending_edus_keyed_by_dest = edus_keyed = {} + # In order to batch outgoing EDUs, we delay sending them. This records the time + # when we should send the next batch, by destination. + self.edu_tx_time_by_dest = {} + + self.edu_tx_task_by_dest = {} + LaterGauge( "synapse_federation_transaction_queue_pending_pdus", "", @@ -397,7 +406,23 @@ def send_edu(self, edu, key): else: self.pending_edus_by_dest.setdefault(edu.destination, []).append(edu) - self._attempt_new_transaction(edu.destination) + destination = edu.destination + + if destination not in self.edu_tx_time_by_dest: + txtime = self.clock.time() + EDU_BATCH_TIME * 1000 + self.edu_tx_time_by_dest[destination] = txtime + + if destination in self.edu_tx_task_by_dest: + # we already have a job queued to send EDUs to this destination + return + + def send_edus(): + del self.edu_tx_task_by_dest[destination] + self._attempt_new_transaction(destination) + + self.edu_tx_task_by_dest[destination] = self.clock.call_later( + EDU_BATCH_TIME, send_edus, + ) def send_device_messages(self, destination): if destination == self.server_name: @@ -422,6 +447,7 @@ def _attempt_new_transaction(self, destination): Returns: None """ + # list of (pending_pdu, deferred, order) if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending @@ -475,18 +501,29 @@ def _transaction_transmission_loop(self, destination): if leftover_pdus: self.pending_pdus_by_dest[destination] = leftover_pdus - pending_edus = self.pending_edus_by_dest.pop(destination, []) + # if we have PDUs to send, we may as well send EDUs too. Otherwise, + # we only send EDUs if their delay is up + if destination in self.edu_tx_time_by_dest and ( + pending_pdus or + self.clock.time() > self.edu_tx_time_by_dest[destination] + ): + del self.edu_tx_time_by_dest[destination] - # We can only include at most 100 EDUs per transactions - pending_edus, leftover_edus = pending_edus[:100], pending_edus[100:] - if leftover_edus: - self.pending_edus_by_dest[destination] = leftover_edus + pending_edus = self.pending_edus_by_dest.pop(destination, []) - pending_presence = self.pending_presence_by_dest.pop(destination, {}) + # We can only include at most 100 EDUs per transactions + pending_edus, leftover_edus = pending_edus[:100], pending_edus[100:] + if leftover_edus: + self.edu_tx_time_by_dest[destination] = self.clock.time() + self.pending_edus_by_dest[destination] = leftover_edus - pending_edus.extend( - self.pending_edus_keyed_by_dest.pop(destination, {}).values() - ) + pending_edus.extend( + self.pending_edus_keyed_by_dest.pop(destination, {}).values() + ) + else: + pending_edus = [] + + pending_presence = self.pending_presence_by_dest.pop(destination, {}) pending_edus.extend(device_message_edus) if pending_presence: From 73c6630718de6950b723c18e25eb7c316f08b608 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 19 Mar 2019 12:17:28 +0000 Subject: [PATCH 072/178] Revert "Reinstate EDU-batching hacks" This reverts commit ed8ccc37377f5ffa0d7d7365747c8897aea6a489. --- synapse/federation/transaction_queue.py | 57 +++++-------------------- 1 file changed, 10 insertions(+), 47 deletions(-) diff --git a/synapse/federation/transaction_queue.py b/synapse/federation/transaction_queue.py index 549bc944a6..e5e42c647d 100644 --- a/synapse/federation/transaction_queue.py +++ b/synapse/federation/transaction_queue.py @@ -66,9 +66,6 @@ ["type"], ) -# number of seconds to wait to batch up outgoing EDUs -EDU_BATCH_TIME = 5.0 - class TransactionQueue(object): """This class makes sure we only have one transaction in flight at @@ -122,12 +119,6 @@ def __init__(self, hs): # Map of destination -> (edu_type, key) -> Edu self.pending_edus_keyed_by_dest = edus_keyed = {} - # In order to batch outgoing EDUs, we delay sending them. This records the time - # when we should send the next batch, by destination. - self.edu_tx_time_by_dest = {} - - self.edu_tx_task_by_dest = {} - LaterGauge( "synapse_federation_transaction_queue_pending_pdus", "", @@ -406,23 +397,7 @@ def send_edu(self, edu, key): else: self.pending_edus_by_dest.setdefault(edu.destination, []).append(edu) - destination = edu.destination - - if destination not in self.edu_tx_time_by_dest: - txtime = self.clock.time() + EDU_BATCH_TIME * 1000 - self.edu_tx_time_by_dest[destination] = txtime - - if destination in self.edu_tx_task_by_dest: - # we already have a job queued to send EDUs to this destination - return - - def send_edus(): - del self.edu_tx_task_by_dest[destination] - self._attempt_new_transaction(destination) - - self.edu_tx_task_by_dest[destination] = self.clock.call_later( - EDU_BATCH_TIME, send_edus, - ) + self._attempt_new_transaction(edu.destination) def send_device_messages(self, destination): if destination == self.server_name: @@ -447,7 +422,6 @@ def _attempt_new_transaction(self, destination): Returns: None """ - # list of (pending_pdu, deferred, order) if destination in self.pending_transactions: # XXX: pending_transactions can get stuck on by a never-ending @@ -501,30 +475,19 @@ def _transaction_transmission_loop(self, destination): if leftover_pdus: self.pending_pdus_by_dest[destination] = leftover_pdus - # if we have PDUs to send, we may as well send EDUs too. Otherwise, - # we only send EDUs if their delay is up - if destination in self.edu_tx_time_by_dest and ( - pending_pdus or - self.clock.time() > self.edu_tx_time_by_dest[destination] - ): - del self.edu_tx_time_by_dest[destination] - - pending_edus = self.pending_edus_by_dest.pop(destination, []) - - # We can only include at most 100 EDUs per transactions - pending_edus, leftover_edus = pending_edus[:100], pending_edus[100:] - if leftover_edus: - self.edu_tx_time_by_dest[destination] = self.clock.time() - self.pending_edus_by_dest[destination] = leftover_edus + pending_edus = self.pending_edus_by_dest.pop(destination, []) - pending_edus.extend( - self.pending_edus_keyed_by_dest.pop(destination, {}).values() - ) - else: - pending_edus = [] + # We can only include at most 100 EDUs per transactions + pending_edus, leftover_edus = pending_edus[:100], pending_edus[100:] + if leftover_edus: + self.pending_edus_by_dest[destination] = leftover_edus pending_presence = self.pending_presence_by_dest.pop(destination, {}) + pending_edus.extend( + self.pending_edus_keyed_by_dest.pop(destination, {}).values() + ) + pending_edus.extend(device_message_edus) if pending_presence: pending_edus.append( From 233b61ac61f39b6c107c14dd399a2ac67d1cfc38 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 2 Apr 2019 13:51:37 +0100 Subject: [PATCH 073/178] Remove spurious changelog files from hotfixes The relevant patches are now in develop thanks to https://github.com/matrix-org/synapse/pull/4816. --- changelog.d/4719.misc | 1 - changelog.d/4769.misc | 1 - 2 files changed, 2 deletions(-) delete mode 100644 changelog.d/4719.misc delete mode 100644 changelog.d/4769.misc diff --git a/changelog.d/4719.misc b/changelog.d/4719.misc deleted file mode 100644 index 8bc536ab66..0000000000 --- a/changelog.d/4719.misc +++ /dev/null @@ -1 +0,0 @@ -Add more debug for membership syncing issues. diff --git a/changelog.d/4769.misc b/changelog.d/4769.misc deleted file mode 100644 index 89144b5425..0000000000 --- a/changelog.d/4769.misc +++ /dev/null @@ -1 +0,0 @@ -Add more debug for #4422. From ec94d6a590e37fb5be4b802579d6e002766e2751 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Apr 2019 19:21:32 +0100 Subject: [PATCH 074/178] VersionRestServlet doesn't take a param --- synapse/app/client_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index 1e9e686107..864f1eac48 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -114,7 +114,7 @@ def _listen_http(self, listener_config): KeyChangesServlet(self).register(resource) VoipRestServlet(self).register(resource) PushRuleRestServlet(self).register(resource) - VersionsRestServlet(self).register(resource) + VersionsRestServlet().register(resource) resources.update({ "/_matrix/client": resource, From aadba440da3ac124a4ecfb13da0fba23a51efbf8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Apr 2019 19:23:21 +0100 Subject: [PATCH 075/178] Point pusher to new box --- synapse/push/httppusher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 495ffbc3bf..6e40863e78 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -109,7 +109,7 @@ def __init__(self, hs, pusherdict): self.url = self.data['url'] self.url = self.url.replace( "https://matrix.org/_matrix/push/v1/notify", - "http://http-priv.matrix.org/_matrix/push/v1/notify", + "http://10.101.0.14/_matrix/push/v1/notify", ) self.http_client = hs.get_simple_http_client() self.data_minus_url = {} From fc5be50d561dcf0f069c5b4920faa0dfd3a962e2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 16 Apr 2019 15:16:57 +0100 Subject: [PATCH 076/178] skip send without trailing slash --- synapse/federation/transport/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index e424c40fdf..de888cc12f 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -173,7 +173,7 @@ def send_transaction(self, transaction, json_data_callback=None): # generated by the json_data_callback. json_data = transaction.get_dict() - path = _create_v1_path("/send/%s", transaction.transaction_id) + path = _create_v1_path("/send/%s/", transaction.transaction_id) response = yield self.client.put_json( transaction.destination, @@ -182,7 +182,7 @@ def send_transaction(self, transaction, json_data_callback=None): json_data_callback=json_data_callback, long_retries=True, backoff_on_404=True, # If we get a 404 the other side has gone - try_trailing_slash_on_400=True, + # try_trailing_slash_on_400=True, ) defer.returnValue(response) From 8699f380f035550ca05d50eadb1fd03fab5ec86b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 4 Jun 2019 12:14:41 +0100 Subject: [PATCH 077/178] hotfix RetryLimiter --- synapse/util/retryutils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index f6dfa77d8f..1a77456498 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -97,7 +97,12 @@ def get_retry_limiter(destination, clock, store, ignore_backoff=False, **kwargs) defer.returnValue( RetryDestinationLimiter( - destination, clock, store, retry_interval, backoff_on_failure, **kwargs + destination, + clock, + store, + retry_interval, + backoff_on_failure=backoff_on_failure, + **kwargs ) ) From d90b0946ed775ca228895dd9f7e63bd16bed6391 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 13 Aug 2019 18:05:06 +0100 Subject: [PATCH 078/178] Temporary fix to ensure kde can contact matrix.org if stuff breaks --- synapse/http/federation/well_known_resolver.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/http/federation/well_known_resolver.py b/synapse/http/federation/well_known_resolver.py index d2866ff67d..d4bbf057e2 100644 --- a/synapse/http/federation/well_known_resolver.py +++ b/synapse/http/federation/well_known_resolver.py @@ -79,6 +79,10 @@ def get_well_known(self, server_name): Returns: Deferred[WellKnownLookupResult]: The result of the lookup """ + + if server_name == b"kde.org": + return WellKnownLookupResult(delegated_server=b"kde.modular.im:443") + try: result = self._well_known_cache[server_name] except KeyError: From 1ceeccb76987cfcc833498a07f866b461f451467 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 6 Sep 2019 13:00:34 +0100 Subject: [PATCH 079/178] Move get_threepid_validation_session into RegistrationWorkerStore --- synapse/storage/registration.py | 151 ++++++++++++++++---------------- 1 file changed, 76 insertions(+), 75 deletions(-) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 2d3c7e2dc9..fae3d92cc6 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -614,6 +614,82 @@ def get_user_deactivated_status(self, user_id): # Convert the integer into a boolean. return res == 1 + def validate_threepid_session(self, session_id, client_secret, token, current_ts): + """Attempt to validate a threepid session using a token + + Args: + session_id (str): The id of a validation session + client_secret (str): A unique string provided by the client to + help identify this validation attempt + token (str): A validation token + current_ts (int): The current unix time in milliseconds. Used for + checking token expiry status + + Returns: + deferred str|None: A str representing a link to redirect the user + to if there is one. + """ + + # Insert everything into a transaction in order to run atomically + def validate_threepid_session_txn(txn): + row = self._simple_select_one_txn( + txn, + table="threepid_validation_session", + keyvalues={"session_id": session_id}, + retcols=["client_secret", "validated_at"], + allow_none=True, + ) + + if not row: + raise ThreepidValidationError(400, "Unknown session_id") + retrieved_client_secret = row["client_secret"] + validated_at = row["validated_at"] + + if retrieved_client_secret != client_secret: + raise ThreepidValidationError( + 400, "This client_secret does not match the provided session_id" + ) + + row = self._simple_select_one_txn( + txn, + table="threepid_validation_token", + keyvalues={"session_id": session_id, "token": token}, + retcols=["expires", "next_link"], + allow_none=True, + ) + + if not row: + raise ThreepidValidationError( + 400, "Validation token not found or has expired" + ) + expires = row["expires"] + next_link = row["next_link"] + + # If the session is already validated, no need to revalidate + if validated_at: + return next_link + + if expires <= current_ts: + raise ThreepidValidationError( + 400, "This token has expired. Please request a new one" + ) + + # Looks good. Validate the session + self._simple_update_txn( + txn, + table="threepid_validation_session", + keyvalues={"session_id": session_id}, + updatevalues={"validated_at": self.clock.time_msec()}, + ) + + return next_link + + # Return next_link if it exists + return self.runInteraction( + "validate_threepid_session_txn", validate_threepid_session_txn + ) + + class RegistrationStore( RegistrationWorkerStore, background_updates.BackgroundUpdateStore @@ -1136,81 +1212,6 @@ def get_threepid_validation_session_txn(txn): "get_threepid_validation_session", get_threepid_validation_session_txn ) - def validate_threepid_session(self, session_id, client_secret, token, current_ts): - """Attempt to validate a threepid session using a token - - Args: - session_id (str): The id of a validation session - client_secret (str): A unique string provided by the client to - help identify this validation attempt - token (str): A validation token - current_ts (int): The current unix time in milliseconds. Used for - checking token expiry status - - Returns: - deferred str|None: A str representing a link to redirect the user - to if there is one. - """ - - # Insert everything into a transaction in order to run atomically - def validate_threepid_session_txn(txn): - row = self._simple_select_one_txn( - txn, - table="threepid_validation_session", - keyvalues={"session_id": session_id}, - retcols=["client_secret", "validated_at"], - allow_none=True, - ) - - if not row: - raise ThreepidValidationError(400, "Unknown session_id") - retrieved_client_secret = row["client_secret"] - validated_at = row["validated_at"] - - if retrieved_client_secret != client_secret: - raise ThreepidValidationError( - 400, "This client_secret does not match the provided session_id" - ) - - row = self._simple_select_one_txn( - txn, - table="threepid_validation_token", - keyvalues={"session_id": session_id, "token": token}, - retcols=["expires", "next_link"], - allow_none=True, - ) - - if not row: - raise ThreepidValidationError( - 400, "Validation token not found or has expired" - ) - expires = row["expires"] - next_link = row["next_link"] - - # If the session is already validated, no need to revalidate - if validated_at: - return next_link - - if expires <= current_ts: - raise ThreepidValidationError( - 400, "This token has expired. Please request a new one" - ) - - # Looks good. Validate the session - self._simple_update_txn( - txn, - table="threepid_validation_session", - keyvalues={"session_id": session_id}, - updatevalues={"validated_at": self.clock.time_msec()}, - ) - - return next_link - - # Return next_link if it exists - return self.runInteraction( - "validate_threepid_session_txn", validate_threepid_session_txn - ) - def upsert_threepid_validation_session( self, medium, From e0eef473158d8b60bbea6fb130cc89796fc3e606 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 11 Sep 2019 11:59:45 +0100 Subject: [PATCH 080/178] Fix existing v2 identity server calls (MSC2140) (#6013) Two things I missed while implementing [MSC2140](https://github.com/matrix-org/matrix-doc/pull/2140/files#diff-c03a26de5ac40fb532de19cb7fc2aaf7R80). 1. Access tokens should be provided to the identity server as `access_token`, not `id_access_token`, even though the homeserver may accept the tokens as `id_access_token`. 2. Access tokens must be sent to the identity server in a query parameter, the JSON body is not allowed. We now send the access token as part of an `Authorization: ...` header, which fixes both things. The breaking code was added in https://github.com/matrix-org/synapse/pull/5892 Sytest PR: https://github.com/matrix-org/sytest/pull/697 --- changelog.d/6013.misc | 1 + synapse/handlers/identity.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 changelog.d/6013.misc diff --git a/changelog.d/6013.misc b/changelog.d/6013.misc new file mode 100644 index 0000000000..939fe8c655 --- /dev/null +++ b/changelog.d/6013.misc @@ -0,0 +1 @@ +Compatibility with v2 Identity Service APIs other than /lookup. \ No newline at end of file diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index f6d1d1717e..73fe98f296 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -74,6 +74,25 @@ def _extract_items_from_creds_dict(self, creds): id_access_token = creds.get("id_access_token") return client_secret, id_server, id_access_token + def create_id_access_token_header(self, id_access_token): + """Create an Authorization header for passing to SimpleHttpClient as the header value + of an HTTP request. + + Args: + id_access_token (str): An identity server access token. + + Returns: + list[str]: The ascii-encoded bearer token encased in a list. + """ + # Prefix with Bearer + bearer_token = "Bearer %s" % id_access_token + + # Encode headers to standard ascii + bearer_token.encode("ascii") + + # Return as a list as that's how SimpleHttpClient takes header values + return [bearer_token] + @defer.inlineCallbacks def threepid_from_creds(self, id_server, creds): """ @@ -149,15 +168,20 @@ def bind_threepid(self, creds, mxid, use_v2=True): use_v2 = False # Decide which API endpoint URLs to use + headers = {} bind_data = {"sid": creds["sid"], "client_secret": client_secret, "mxid": mxid} if use_v2: bind_url = "https://%s/_matrix/identity/v2/3pid/bind" % (id_server,) - bind_data["id_access_token"] = id_access_token + headers["Authorization"] = self.create_id_access_token_header( + id_access_token + ) else: bind_url = "https://%s/_matrix/identity/api/v1/3pid/bind" % (id_server,) try: - data = yield self.http_client.post_json_get_json(bind_url, bind_data) + data = yield self.http_client.post_json_get_json( + bind_url, bind_data, headers=headers + ) logger.debug("bound threepid %r to %s", creds, mxid) # Remember where we bound the threepid From ee91c69ef792c3db7b4438d17e38f343f9c10b72 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 13 Sep 2019 14:44:48 +0100 Subject: [PATCH 081/178] Fix m.federate bug --- synapse/handlers/stats.py | 2 +- synapse/storage/stats.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index 921735edb3..4a962d97c9 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -260,7 +260,7 @@ def _handle_deltas(self, deltas): room_stats_delta["local_users_in_room"] += delta elif typ == EventTypes.Create: - room_state["is_federatable"] = event_content.get("m.federate", True) + room_state["is_federatable"] = event_content.get("m.federate", True) is True if sender and self.is_mine_id(sender): user_to_stats_deltas.setdefault(sender, Counter())[ "rooms_created" diff --git a/synapse/storage/stats.py b/synapse/storage/stats.py index 6560173c08..8ba7051086 100644 --- a/synapse/storage/stats.py +++ b/synapse/storage/stats.py @@ -823,7 +823,7 @@ def _fetch_current_state_stats(txn): elif event.type == EventTypes.CanonicalAlias: room_state["canonical_alias"] = event.content.get("alias") elif event.type == EventTypes.Create: - room_state["is_federatable"] = event.content.get("m.federate", True) + room_state["is_federatable"] = event.content.get("m.federate", True) is True yield self.update_room_state(room_id, room_state) From e01026d84d74c78fa6a0411c2597cf33ec8fbde5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Sep 2019 13:53:37 +0100 Subject: [PATCH 082/178] Revert "Fix existing v2 identity server calls (MSC2140) (#6013)" This has now been merged into develop (3505ffcda) so we don't need this cherry-picked commit. This reverts commit e0eef473158d8b60bbea6fb130cc89796fc3e606. --- changelog.d/6013.misc | 1 - synapse/handlers/identity.py | 28 ++-------------------------- 2 files changed, 2 insertions(+), 27 deletions(-) delete mode 100644 changelog.d/6013.misc diff --git a/changelog.d/6013.misc b/changelog.d/6013.misc deleted file mode 100644 index 939fe8c655..0000000000 --- a/changelog.d/6013.misc +++ /dev/null @@ -1 +0,0 @@ -Compatibility with v2 Identity Service APIs other than /lookup. \ No newline at end of file diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 73fe98f296..f6d1d1717e 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -74,25 +74,6 @@ def _extract_items_from_creds_dict(self, creds): id_access_token = creds.get("id_access_token") return client_secret, id_server, id_access_token - def create_id_access_token_header(self, id_access_token): - """Create an Authorization header for passing to SimpleHttpClient as the header value - of an HTTP request. - - Args: - id_access_token (str): An identity server access token. - - Returns: - list[str]: The ascii-encoded bearer token encased in a list. - """ - # Prefix with Bearer - bearer_token = "Bearer %s" % id_access_token - - # Encode headers to standard ascii - bearer_token.encode("ascii") - - # Return as a list as that's how SimpleHttpClient takes header values - return [bearer_token] - @defer.inlineCallbacks def threepid_from_creds(self, id_server, creds): """ @@ -168,20 +149,15 @@ def bind_threepid(self, creds, mxid, use_v2=True): use_v2 = False # Decide which API endpoint URLs to use - headers = {} bind_data = {"sid": creds["sid"], "client_secret": client_secret, "mxid": mxid} if use_v2: bind_url = "https://%s/_matrix/identity/v2/3pid/bind" % (id_server,) - headers["Authorization"] = self.create_id_access_token_header( - id_access_token - ) + bind_data["id_access_token"] = id_access_token else: bind_url = "https://%s/_matrix/identity/api/v1/3pid/bind" % (id_server,) try: - data = yield self.http_client.post_json_get_json( - bind_url, bind_data, headers=headers - ) + data = yield self.http_client.post_json_get_json(bind_url, bind_data) logger.debug("bound threepid %r to %s", creds, mxid) # Remember where we bound the threepid From 05241b30316584cbb341cc8b995b8d6aefb827d8 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 18 Sep 2019 13:54:57 +0100 Subject: [PATCH 083/178] Revert "Fix m.federate bug" This has now been merged into develop (142c9325c) so we no longer need this cherry-picked commit. This reverts commit ee91c69ef792c3db7b4438d17e38f343f9c10b72. --- synapse/handlers/stats.py | 2 +- synapse/storage/stats.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index 4a962d97c9..921735edb3 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -260,7 +260,7 @@ def _handle_deltas(self, deltas): room_stats_delta["local_users_in_room"] += delta elif typ == EventTypes.Create: - room_state["is_federatable"] = event_content.get("m.federate", True) is True + room_state["is_federatable"] = event_content.get("m.federate", True) if sender and self.is_mine_id(sender): user_to_stats_deltas.setdefault(sender, Counter())[ "rooms_created" diff --git a/synapse/storage/stats.py b/synapse/storage/stats.py index 8ba7051086..6560173c08 100644 --- a/synapse/storage/stats.py +++ b/synapse/storage/stats.py @@ -823,7 +823,7 @@ def _fetch_current_state_stats(txn): elif event.type == EventTypes.CanonicalAlias: room_state["canonical_alias"] = event.content.get("alias") elif event.type == EventTypes.Create: - room_state["is_federatable"] = event.content.get("m.federate", True) is True + room_state["is_federatable"] = event.content.get("m.federate", True) yield self.update_room_state(room_id, room_state) From 721086a291b7d187725ab3314dc1a4f11bd00f46 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Sep 2019 16:00:00 +0100 Subject: [PATCH 084/178] Awful hackery to try to get the fed sender to keep up Basically, if the federation sender starts getting behind, insert some sleeps into the transaction transmission code to give the fed sender a chance to catch up. Might have to experiment a bit with the numbers. --- changelog.d/6126.feature | 1 + synapse/federation/sender/__init__.py | 18 ++++++++++++++++++ .../federation/sender/per_destination_queue.py | 5 +++++ .../federation/sender/transaction_manager.py | 4 ++++ 4 files changed, 28 insertions(+) create mode 100644 changelog.d/6126.feature diff --git a/changelog.d/6126.feature b/changelog.d/6126.feature new file mode 100644 index 0000000000..1207ba6206 --- /dev/null +++ b/changelog.d/6126.feature @@ -0,0 +1 @@ +Group events into larger federation transactions at times of high traffic. diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index d46f4aaeb1..497485fac2 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -152,9 +152,24 @@ def notify_new_events(self, current_id): @defer.inlineCallbacks def _process_event_queue_loop(self): + loop_start_time = self.clock.time_msec() try: self._is_processing = True while True: + # if we've been going around this loop for a long time without + # catching up, deprioritise transaction transmission. This should mean + # that events get batched into fewer transactions, which is more + # efficient, and hence give us a chance to catch up + if ( + self.clock.time_msec() - loop_start_time > 60 * 1000 + and not self._transaction_manager.deprioritise_transmission + ): + logger.warning( + "Event processing loop is getting behind: deprioritising " + "transaction transmission" + ) + self._transaction_manager.deprioritise_transmission = True + last_token = yield self.store.get_federation_out_pos("events") next_token, events = yield self.store.get_all_new_events_stream( last_token, self._last_poked_id, limit=100 @@ -251,6 +266,9 @@ def handle_room_events(events): finally: self._is_processing = False + if self._transaction_manager.deprioritise_transmission: + logger.info("Event queue caught up: re-prioritising transmission") + self._transaction_manager.deprioritise_transmission = False def _send_pdu(self, pdu, destinations): # We loop through all destinations to see whether we already have diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index fad980b893..b890aaf840 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -189,6 +189,11 @@ def _transaction_transmission_loop(self): pending_pdus = [] while True: + if self._transaction_manager.deprioritise_transmission: + # if the event-processing loop has got behind, sleep to give it + # a chance to catch up + yield self._clock.sleep(2) + # We have to keep 2 free slots for presence and rr_edus limit = MAX_EDUS_PER_TRANSACTION - 2 diff --git a/synapse/federation/sender/transaction_manager.py b/synapse/federation/sender/transaction_manager.py index 5b6c79c51a..69679dbf65 100644 --- a/synapse/federation/sender/transaction_manager.py +++ b/synapse/federation/sender/transaction_manager.py @@ -49,6 +49,10 @@ def __init__(self, hs): # HACK to get unique tx id self._next_txn_id = int(self.clock.time_msec()) + # the federation sender sometimes sets this to delay transaction transmission, + # if the sender gets behind. + self.deprioritise_transmission = False + @measure_func("_send_new_transaction") @defer.inlineCallbacks def send_new_transaction(self, destination, pending_pdus, pending_edus): From cb217d5d60c9e006357b0d37247d5acb3daae84a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 3 Oct 2019 17:05:24 +0100 Subject: [PATCH 085/178] Revert "Awful hackery to try to get the fed sender to keep up" This reverts commit 721086a291b7d187725ab3314dc1a4f11bd00f46. This didn't help. --- changelog.d/6126.feature | 1 - synapse/federation/sender/__init__.py | 18 ------------------ .../federation/sender/per_destination_queue.py | 5 ----- .../federation/sender/transaction_manager.py | 4 ---- 4 files changed, 28 deletions(-) delete mode 100644 changelog.d/6126.feature diff --git a/changelog.d/6126.feature b/changelog.d/6126.feature deleted file mode 100644 index 1207ba6206..0000000000 --- a/changelog.d/6126.feature +++ /dev/null @@ -1 +0,0 @@ -Group events into larger federation transactions at times of high traffic. diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 497485fac2..d46f4aaeb1 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -152,24 +152,9 @@ def notify_new_events(self, current_id): @defer.inlineCallbacks def _process_event_queue_loop(self): - loop_start_time = self.clock.time_msec() try: self._is_processing = True while True: - # if we've been going around this loop for a long time without - # catching up, deprioritise transaction transmission. This should mean - # that events get batched into fewer transactions, which is more - # efficient, and hence give us a chance to catch up - if ( - self.clock.time_msec() - loop_start_time > 60 * 1000 - and not self._transaction_manager.deprioritise_transmission - ): - logger.warning( - "Event processing loop is getting behind: deprioritising " - "transaction transmission" - ) - self._transaction_manager.deprioritise_transmission = True - last_token = yield self.store.get_federation_out_pos("events") next_token, events = yield self.store.get_all_new_events_stream( last_token, self._last_poked_id, limit=100 @@ -266,9 +251,6 @@ def handle_room_events(events): finally: self._is_processing = False - if self._transaction_manager.deprioritise_transmission: - logger.info("Event queue caught up: re-prioritising transmission") - self._transaction_manager.deprioritise_transmission = False def _send_pdu(self, pdu, destinations): # We loop through all destinations to see whether we already have diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index b890aaf840..fad980b893 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -189,11 +189,6 @@ def _transaction_transmission_loop(self): pending_pdus = [] while True: - if self._transaction_manager.deprioritise_transmission: - # if the event-processing loop has got behind, sleep to give it - # a chance to catch up - yield self._clock.sleep(2) - # We have to keep 2 free slots for presence and rr_edus limit = MAX_EDUS_PER_TRANSACTION - 2 diff --git a/synapse/federation/sender/transaction_manager.py b/synapse/federation/sender/transaction_manager.py index 69679dbf65..5b6c79c51a 100644 --- a/synapse/federation/sender/transaction_manager.py +++ b/synapse/federation/sender/transaction_manager.py @@ -49,10 +49,6 @@ def __init__(self, hs): # HACK to get unique tx id self._next_txn_id = int(self.clock.time_msec()) - # the federation sender sometimes sets this to delay transaction transmission, - # if the sender gets behind. - self.deprioritise_transmission = False - @measure_func("_send_new_transaction") @defer.inlineCallbacks def send_new_transaction(self, destination, pending_pdus, pending_edus): From b852a8247d1132fae125c3fb813023b6ec3f6cb3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 27 Sep 2019 16:00:00 +0100 Subject: [PATCH 086/178] Awful hackery to try to get the fed sender to keep up Basically, if the federation sender starts getting behind, insert some sleeps into the transaction transmission code to give the fed sender a chance to catch up. Might have to experiment a bit with the numbers. --- changelog.d/6126.feature | 1 + synapse/federation/sender/__init__.py | 18 ++++++++++++++++++ .../federation/sender/per_destination_queue.py | 5 +++++ .../federation/sender/transaction_manager.py | 4 ++++ 4 files changed, 28 insertions(+) create mode 100644 changelog.d/6126.feature diff --git a/changelog.d/6126.feature b/changelog.d/6126.feature new file mode 100644 index 0000000000..1207ba6206 --- /dev/null +++ b/changelog.d/6126.feature @@ -0,0 +1 @@ +Group events into larger federation transactions at times of high traffic. diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 2b2ee8612a..f23bbf0e1f 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -152,9 +152,24 @@ def notify_new_events(self, current_id): @defer.inlineCallbacks def _process_event_queue_loop(self): + loop_start_time = self.clock.time_msec() try: self._is_processing = True while True: + # if we've been going around this loop for a long time without + # catching up, deprioritise transaction transmission. This should mean + # that events get batched into fewer transactions, which is more + # efficient, and hence give us a chance to catch up + if ( + self.clock.time_msec() - loop_start_time > 60 * 1000 + and not self._transaction_manager.deprioritise_transmission + ): + logger.warning( + "Event processing loop is getting behind: deprioritising " + "transaction transmission" + ) + self._transaction_manager.deprioritise_transmission = True + last_token = yield self.store.get_federation_out_pos("events") next_token, events = yield self.store.get_all_new_events_stream( last_token, self._last_poked_id, limit=100 @@ -252,6 +267,9 @@ def handle_room_events(events): finally: self._is_processing = False + if self._transaction_manager.deprioritise_transmission: + logger.info("Event queue caught up: re-prioritising transmission") + self._transaction_manager.deprioritise_transmission = False def _send_pdu(self, pdu, destinations): # We loop through all destinations to see whether we already have diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index fad980b893..b890aaf840 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -189,6 +189,11 @@ def _transaction_transmission_loop(self): pending_pdus = [] while True: + if self._transaction_manager.deprioritise_transmission: + # if the event-processing loop has got behind, sleep to give it + # a chance to catch up + yield self._clock.sleep(2) + # We have to keep 2 free slots for presence and rr_edus limit = MAX_EDUS_PER_TRANSACTION - 2 diff --git a/synapse/federation/sender/transaction_manager.py b/synapse/federation/sender/transaction_manager.py index 5b6c79c51a..69679dbf65 100644 --- a/synapse/federation/sender/transaction_manager.py +++ b/synapse/federation/sender/transaction_manager.py @@ -49,6 +49,10 @@ def __init__(self, hs): # HACK to get unique tx id self._next_txn_id = int(self.clock.time_msec()) + # the federation sender sometimes sets this to delay transaction transmission, + # if the sender gets behind. + self.deprioritise_transmission = False + @measure_func("_send_new_transaction") @defer.inlineCallbacks def send_new_transaction(self, destination, pending_pdus, pending_edus): From 15b2a50817348c8e25a8e5f420b3dfd20451c6d6 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 11 Oct 2019 09:15:56 +0100 Subject: [PATCH 087/178] Add some randomness to the high-cpu backoff hack --- synapse/federation/sender/__init__.py | 4 ++-- synapse/federation/sender/per_destination_queue.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index f23bbf0e1f..788b26446d 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -165,8 +165,8 @@ def _process_event_queue_loop(self): and not self._transaction_manager.deprioritise_transmission ): logger.warning( - "Event processing loop is getting behind: deprioritising " - "transaction transmission" + "Event queue is getting behind: deprioritising transaction " + "transmission" ) self._transaction_manager.deprioritise_transmission = True diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index b890aaf840..69a6f47b7b 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -15,6 +15,7 @@ # limitations under the License. import datetime import logging +import random from prometheus_client import Counter @@ -36,6 +37,8 @@ # This is defined in the Matrix spec and enforced by the receiver. MAX_EDUS_PER_TRANSACTION = 100 +DEPRIORITISE_SLEEP_TIME = 10 + logger = logging.getLogger(__name__) @@ -191,8 +194,13 @@ def _transaction_transmission_loop(self): while True: if self._transaction_manager.deprioritise_transmission: # if the event-processing loop has got behind, sleep to give it - # a chance to catch up - yield self._clock.sleep(2) + # a chance to catch up. Add some randomness so that the transmitters + # don't all wake up in sync. + sleeptime = random.uniform( + DEPRIORITISE_SLEEP_TIME, DEPRIORITISE_SLEEP_TIME * 2 + ) + logger.info("TX [%s]: sleeping for %f seconds", sleeptime) + yield self._clock.sleep(sleeptime) # We have to keep 2 free slots for presence and rr_edus limit = MAX_EDUS_PER_TRANSACTION - 2 From 28889d8da5b1ca8c7106e8ace246220e96af9f59 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 11 Oct 2019 09:57:18 +0100 Subject: [PATCH 088/178] fix logging --- synapse/federation/sender/per_destination_queue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index 69a6f47b7b..818c102799 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -199,7 +199,9 @@ def _transaction_transmission_loop(self): sleeptime = random.uniform( DEPRIORITISE_SLEEP_TIME, DEPRIORITISE_SLEEP_TIME * 2 ) - logger.info("TX [%s]: sleeping for %f seconds", sleeptime) + logger.info( + "TX [%s]: sleeping for %f seconds", self._destination, sleeptime + ) yield self._clock.sleep(sleeptime) # We have to keep 2 free slots for presence and rr_edus From 5407e69732b42e47d084f3ac179eedbecd1f78be Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Nov 2019 12:04:19 +0000 Subject: [PATCH 089/178] Change /push/v1/notify IP to 10.103.0.7 --- synapse/push/httppusher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 338267c82d..026575278c 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -105,7 +105,7 @@ def __init__(self, hs, pusherdict): self.url = self.data["url"] self.url = self.url.replace( "https://matrix.org/_matrix/push/v1/notify", - "http://10.101.0.14/_matrix/push/v1/notify", + "http://10.103.0.7/_matrix/push/v1/notify", ) self.http_client = hs.get_proxied_http_client() self.data_minus_url = {} From 508e0f9310abedb4d10602e7bf7e57512900d80c Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 26 Nov 2019 12:15:46 +0000 Subject: [PATCH 090/178] 1.6.0 --- CHANGES.md | 6 ++++++ debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index d26bc7a86f..f25627442e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +Synapse 1.6.0 (2019-11-26) +========================== + +No significant changes. + + Synapse 1.6.0rc2 (2019-11-25) ============================= diff --git a/debian/changelog b/debian/changelog index c4415f460a..82dae017f1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.6.0) stable; urgency=medium + + * New synapse release 1.6.0. + + -- Synapse Packaging team Tue, 26 Nov 2019 12:15:40 +0000 + matrix-synapse-py3 (1.5.1) stable; urgency=medium * New synapse release 1.5.1. diff --git a/synapse/__init__.py b/synapse/__init__.py index 051c83774e..53eedc0048 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -36,7 +36,7 @@ except ImportError: pass -__version__ = "1.6.0rc2" +__version__ = "1.6.0" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From aebe20c4524da46241d1d8a0e87a9fa6e7b5056a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 26 Nov 2019 13:10:09 +0000 Subject: [PATCH 091/178] Fix phone home stats (#6418) Fix phone home stats --- changelog.d/6418.bugfix | 1 + synapse/app/homeserver.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/6418.bugfix diff --git a/changelog.d/6418.bugfix b/changelog.d/6418.bugfix new file mode 100644 index 0000000000..a1f488d3a2 --- /dev/null +++ b/changelog.d/6418.bugfix @@ -0,0 +1 @@ +Fix phone home stats reporting. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 73e2c29d06..883b3fb70b 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -585,7 +585,7 @@ def profiled(*args, **kargs): def performance_stats_init(): _stats_process.clear() _stats_process.append( - (int(hs.get_clock().time(), resource.getrusage(resource.RUSAGE_SELF))) + (int(hs.get_clock().time()), resource.getrusage(resource.RUSAGE_SELF)) ) def start_phone_stats_home(): From 4d02bfd6e1e6388a0c67c618570fc35c814d8017 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Jan 2020 23:02:58 +0000 Subject: [PATCH 092/178] a bit of debugging for media storage providers --- synapse/rest/media/v1/media_storage.py | 1 + synapse/rest/media/v1/storage_provider.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/synapse/rest/media/v1/media_storage.py b/synapse/rest/media/v1/media_storage.py index 3b87717a5a..683a79c966 100644 --- a/synapse/rest/media/v1/media_storage.py +++ b/synapse/rest/media/v1/media_storage.py @@ -148,6 +148,7 @@ def fetch_media(self, file_info): for provider in self.storage_providers: res = yield provider.fetch(path, file_info) if res: + logger.debug("Streaming %s from %s", path, provider) return res return None diff --git a/synapse/rest/media/v1/storage_provider.py b/synapse/rest/media/v1/storage_provider.py index 37687ea7f4..858680be26 100644 --- a/synapse/rest/media/v1/storage_provider.py +++ b/synapse/rest/media/v1/storage_provider.py @@ -77,6 +77,9 @@ def __init__(self, backend, store_local, store_synchronous, store_remote): self.store_synchronous = store_synchronous self.store_remote = store_remote + def __str__(self): + return "StorageProviderWrapper[%s]" % (self.backend,) + def store_file(self, path, file_info): if not file_info.server_name and not self.store_local: return defer.succeed(None) @@ -114,6 +117,9 @@ def __init__(self, hs, config): self.cache_directory = hs.config.media_store_path self.base_directory = config + def __str__(self): + return "FileStorageProviderBackend[%s]" % (self.base_directory,) + def store_file(self, path, file_info): """See StorageProvider.store_file""" From 77a166577a9833bb0df0d2e0bd6631a18a15a8e5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Feb 2020 11:14:19 +0000 Subject: [PATCH 093/178] Allow moving group read APIs to workers (#6866) --- changelog.d/6866.feature | 1 + docs/workers.md | 8 + synapse/app/client_reader.py | 3 + synapse/app/federation_reader.py | 2 + synapse/groups/groups_server.py | 377 ++++---- synapse/handlers/groups_local.py | 270 +++--- synapse/replication/slave/storage/groups.py | 14 +- synapse/server.py | 14 +- .../storage/data_stores/main/group_server.py | 880 +++++++++--------- 9 files changed, 802 insertions(+), 767 deletions(-) create mode 100644 changelog.d/6866.feature diff --git a/changelog.d/6866.feature b/changelog.d/6866.feature new file mode 100644 index 0000000000..256feab6ff --- /dev/null +++ b/changelog.d/6866.feature @@ -0,0 +1 @@ +Add ability to run some group APIs on workers. diff --git a/docs/workers.md b/docs/workers.md index 09a9d8a7b8..82442d6a0a 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -177,8 +177,13 @@ endpoints matching the following regular expressions: ^/_matrix/federation/v1/event_auth/ ^/_matrix/federation/v1/exchange_third_party_invite/ ^/_matrix/federation/v1/send/ + ^/_matrix/federation/v1/get_groups_publicised$ ^/_matrix/key/v2/query +Additionally, the following REST endpoints can be handled for GET requests: + + ^/_matrix/federation/v1/groups/ + The above endpoints should all be routed to the federation_reader worker by the reverse-proxy configuration. @@ -254,10 +259,13 @@ following regular expressions: ^/_matrix/client/(api/v1|r0|unstable)/keys/changes$ ^/_matrix/client/versions$ ^/_matrix/client/(api/v1|r0|unstable)/voip/turnServer$ + ^/_matrix/client/(api/v1|r0|unstable)/joined_groups$ + ^/_matrix/client/(api/v1|r0|unstable)/get_groups_publicised$ Additionally, the following REST endpoints can be handled for GET requests: ^/_matrix/client/(api/v1|r0|unstable)/pushrules/.*$ + ^/_matrix/client/(api/v1|r0|unstable)/groups/.*$ Additionally, the following REST endpoints can be handled, but all requests must be routed to the same instance: diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index ca96da6a4a..7fa91a3b11 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -57,6 +57,7 @@ RoomStateRestServlet, ) from synapse.rest.client.v1.voip import VoipRestServlet +from synapse.rest.client.v2_alpha import groups from synapse.rest.client.v2_alpha.account import ThreepidRestServlet from synapse.rest.client.v2_alpha.keys import KeyChangesServlet, KeyQueryServlet from synapse.rest.client.v2_alpha.register import RegisterRestServlet @@ -124,6 +125,8 @@ def _listen_http(self, listener_config): PushRuleRestServlet(self).register(resource) VersionsRestServlet(self).register(resource) + groups.register_servlets(self, resource) + resources.update({"/_matrix/client": resource}) root_resource = create_resource_tree(resources, NoResource()) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 1f1cea1416..5e17ef1396 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -35,6 +35,7 @@ from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore from synapse.replication.slave.storage.directory import DirectoryStore from synapse.replication.slave.storage.events import SlavedEventStore +from synapse.replication.slave.storage.groups import SlavedGroupServerStore from synapse.replication.slave.storage.keys import SlavedKeyStore from synapse.replication.slave.storage.profile import SlavedProfileStore from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore @@ -66,6 +67,7 @@ class FederationReaderSlavedStore( SlavedEventStore, SlavedKeyStore, SlavedRegistrationStore, + SlavedGroupServerStore, RoomStore, DirectoryStore, SlavedTransactionStore, diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py index 0ec9be3cb5..c106abae21 100644 --- a/synapse/groups/groups_server.py +++ b/synapse/groups/groups_server.py @@ -36,7 +36,7 @@ # TODO: Flairs -class GroupsServerHandler(object): +class GroupsServerWorkerHandler(object): def __init__(self, hs): self.hs = hs self.store = hs.get_datastore() @@ -51,9 +51,6 @@ def __init__(self, hs): self.transport_client = hs.get_federation_transport_client() self.profile_handler = hs.get_profile_handler() - # Ensure attestations get renewed - hs.get_groups_attestation_renewer() - @defer.inlineCallbacks def check_group_is_ours( self, group_id, requester_user_id, and_exists=False, and_is_admin=None @@ -167,6 +164,197 @@ def get_group_summary(self, group_id, requester_user_id): "user": membership_info, } + @defer.inlineCallbacks + def get_group_categories(self, group_id, requester_user_id): + """Get all categories in a group (as seen by user) + """ + yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True) + + categories = yield self.store.get_group_categories(group_id=group_id) + return {"categories": categories} + + @defer.inlineCallbacks + def get_group_category(self, group_id, requester_user_id, category_id): + """Get a specific category in a group (as seen by user) + """ + yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True) + + res = yield self.store.get_group_category( + group_id=group_id, category_id=category_id + ) + + logger.info("group %s", res) + + return res + + @defer.inlineCallbacks + def get_group_roles(self, group_id, requester_user_id): + """Get all roles in a group (as seen by user) + """ + yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True) + + roles = yield self.store.get_group_roles(group_id=group_id) + return {"roles": roles} + + @defer.inlineCallbacks + def get_group_role(self, group_id, requester_user_id, role_id): + """Get a specific role in a group (as seen by user) + """ + yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True) + + res = yield self.store.get_group_role(group_id=group_id, role_id=role_id) + return res + + @defer.inlineCallbacks + def get_group_profile(self, group_id, requester_user_id): + """Get the group profile as seen by requester_user_id + """ + + yield self.check_group_is_ours(group_id, requester_user_id) + + group = yield self.store.get_group(group_id) + + if group: + cols = [ + "name", + "short_description", + "long_description", + "avatar_url", + "is_public", + ] + group_description = {key: group[key] for key in cols} + group_description["is_openly_joinable"] = group["join_policy"] == "open" + + return group_description + else: + raise SynapseError(404, "Unknown group") + + @defer.inlineCallbacks + def get_users_in_group(self, group_id, requester_user_id): + """Get the users in group as seen by requester_user_id. + + The ordering is arbitrary at the moment + """ + + yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True) + + is_user_in_group = yield self.store.is_user_in_group( + requester_user_id, group_id + ) + + user_results = yield self.store.get_users_in_group( + group_id, include_private=is_user_in_group + ) + + chunk = [] + for user_result in user_results: + g_user_id = user_result["user_id"] + is_public = user_result["is_public"] + is_privileged = user_result["is_admin"] + + entry = {"user_id": g_user_id} + + profile = yield self.profile_handler.get_profile_from_cache(g_user_id) + entry.update(profile) + + entry["is_public"] = bool(is_public) + entry["is_privileged"] = bool(is_privileged) + + if not self.is_mine_id(g_user_id): + attestation = yield self.store.get_remote_attestation( + group_id, g_user_id + ) + if not attestation: + continue + + entry["attestation"] = attestation + else: + entry["attestation"] = self.attestations.create_attestation( + group_id, g_user_id + ) + + chunk.append(entry) + + # TODO: If admin add lists of users whose attestations have timed out + + return {"chunk": chunk, "total_user_count_estimate": len(user_results)} + + @defer.inlineCallbacks + def get_invited_users_in_group(self, group_id, requester_user_id): + """Get the users that have been invited to a group as seen by requester_user_id. + + The ordering is arbitrary at the moment + """ + + yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True) + + is_user_in_group = yield self.store.is_user_in_group( + requester_user_id, group_id + ) + + if not is_user_in_group: + raise SynapseError(403, "User not in group") + + invited_users = yield self.store.get_invited_users_in_group(group_id) + + user_profiles = [] + + for user_id in invited_users: + user_profile = {"user_id": user_id} + try: + profile = yield self.profile_handler.get_profile_from_cache(user_id) + user_profile.update(profile) + except Exception as e: + logger.warning("Error getting profile for %s: %s", user_id, e) + user_profiles.append(user_profile) + + return {"chunk": user_profiles, "total_user_count_estimate": len(invited_users)} + + @defer.inlineCallbacks + def get_rooms_in_group(self, group_id, requester_user_id): + """Get the rooms in group as seen by requester_user_id + + This returns rooms in order of decreasing number of joined users + """ + + yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True) + + is_user_in_group = yield self.store.is_user_in_group( + requester_user_id, group_id + ) + + room_results = yield self.store.get_rooms_in_group( + group_id, include_private=is_user_in_group + ) + + chunk = [] + for room_result in room_results: + room_id = room_result["room_id"] + + joined_users = yield self.store.get_users_in_room(room_id) + entry = yield self.room_list_handler.generate_room_entry( + room_id, len(joined_users), with_alias=False, allow_private=True + ) + + if not entry: + continue + + entry["is_public"] = bool(room_result["is_public"]) + + chunk.append(entry) + + chunk.sort(key=lambda e: -e["num_joined_members"]) + + return {"chunk": chunk, "total_room_count_estimate": len(room_results)} + + +class GroupsServerHandler(GroupsServerWorkerHandler): + def __init__(self, hs): + super(GroupsServerHandler, self).__init__(hs) + + # Ensure attestations get renewed + hs.get_groups_attestation_renewer() + @defer.inlineCallbacks def update_group_summary_room( self, group_id, requester_user_id, room_id, category_id, content @@ -229,27 +417,6 @@ def set_group_join_policy(self, group_id, requester_user_id, content): return {} - @defer.inlineCallbacks - def get_group_categories(self, group_id, requester_user_id): - """Get all categories in a group (as seen by user) - """ - yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True) - - categories = yield self.store.get_group_categories(group_id=group_id) - return {"categories": categories} - - @defer.inlineCallbacks - def get_group_category(self, group_id, requester_user_id, category_id): - """Get a specific category in a group (as seen by user) - """ - yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True) - - res = yield self.store.get_group_category( - group_id=group_id, category_id=category_id - ) - - return res - @defer.inlineCallbacks def update_group_category(self, group_id, requester_user_id, category_id, content): """Add/Update a group category @@ -284,24 +451,6 @@ def delete_group_category(self, group_id, requester_user_id, category_id): return {} - @defer.inlineCallbacks - def get_group_roles(self, group_id, requester_user_id): - """Get all roles in a group (as seen by user) - """ - yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True) - - roles = yield self.store.get_group_roles(group_id=group_id) - return {"roles": roles} - - @defer.inlineCallbacks - def get_group_role(self, group_id, requester_user_id, role_id): - """Get a specific role in a group (as seen by user) - """ - yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True) - - res = yield self.store.get_group_role(group_id=group_id, role_id=role_id) - return res - @defer.inlineCallbacks def update_group_role(self, group_id, requester_user_id, role_id, content): """Add/update a role in a group @@ -370,30 +519,6 @@ def delete_group_summary_user(self, group_id, requester_user_id, user_id, role_i return {} - @defer.inlineCallbacks - def get_group_profile(self, group_id, requester_user_id): - """Get the group profile as seen by requester_user_id - """ - - yield self.check_group_is_ours(group_id, requester_user_id) - - group = yield self.store.get_group(group_id) - - if group: - cols = [ - "name", - "short_description", - "long_description", - "avatar_url", - "is_public", - ] - group_description = {key: group[key] for key in cols} - group_description["is_openly_joinable"] = group["join_policy"] == "open" - - return group_description - else: - raise SynapseError(404, "Unknown group") - @defer.inlineCallbacks def update_group_profile(self, group_id, requester_user_id, content): """Update the group profile @@ -412,124 +537,6 @@ def update_group_profile(self, group_id, requester_user_id, content): yield self.store.update_group_profile(group_id, profile) - @defer.inlineCallbacks - def get_users_in_group(self, group_id, requester_user_id): - """Get the users in group as seen by requester_user_id. - - The ordering is arbitrary at the moment - """ - - yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True) - - is_user_in_group = yield self.store.is_user_in_group( - requester_user_id, group_id - ) - - user_results = yield self.store.get_users_in_group( - group_id, include_private=is_user_in_group - ) - - chunk = [] - for user_result in user_results: - g_user_id = user_result["user_id"] - is_public = user_result["is_public"] - is_privileged = user_result["is_admin"] - - entry = {"user_id": g_user_id} - - profile = yield self.profile_handler.get_profile_from_cache(g_user_id) - entry.update(profile) - - entry["is_public"] = bool(is_public) - entry["is_privileged"] = bool(is_privileged) - - if not self.is_mine_id(g_user_id): - attestation = yield self.store.get_remote_attestation( - group_id, g_user_id - ) - if not attestation: - continue - - entry["attestation"] = attestation - else: - entry["attestation"] = self.attestations.create_attestation( - group_id, g_user_id - ) - - chunk.append(entry) - - # TODO: If admin add lists of users whose attestations have timed out - - return {"chunk": chunk, "total_user_count_estimate": len(user_results)} - - @defer.inlineCallbacks - def get_invited_users_in_group(self, group_id, requester_user_id): - """Get the users that have been invited to a group as seen by requester_user_id. - - The ordering is arbitrary at the moment - """ - - yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True) - - is_user_in_group = yield self.store.is_user_in_group( - requester_user_id, group_id - ) - - if not is_user_in_group: - raise SynapseError(403, "User not in group") - - invited_users = yield self.store.get_invited_users_in_group(group_id) - - user_profiles = [] - - for user_id in invited_users: - user_profile = {"user_id": user_id} - try: - profile = yield self.profile_handler.get_profile_from_cache(user_id) - user_profile.update(profile) - except Exception as e: - logger.warning("Error getting profile for %s: %s", user_id, e) - user_profiles.append(user_profile) - - return {"chunk": user_profiles, "total_user_count_estimate": len(invited_users)} - - @defer.inlineCallbacks - def get_rooms_in_group(self, group_id, requester_user_id): - """Get the rooms in group as seen by requester_user_id - - This returns rooms in order of decreasing number of joined users - """ - - yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True) - - is_user_in_group = yield self.store.is_user_in_group( - requester_user_id, group_id - ) - - room_results = yield self.store.get_rooms_in_group( - group_id, include_private=is_user_in_group - ) - - chunk = [] - for room_result in room_results: - room_id = room_result["room_id"] - - joined_users = yield self.store.get_users_in_room(room_id) - entry = yield self.room_list_handler.generate_room_entry( - room_id, len(joined_users), with_alias=False, allow_private=True - ) - - if not entry: - continue - - entry["is_public"] = bool(room_result["is_public"]) - - chunk.append(entry) - - chunk.sort(key=lambda e: -e["num_joined_members"]) - - return {"chunk": chunk, "total_room_count_estimate": len(room_results)} - @defer.inlineCallbacks def add_room_to_group(self, group_id, requester_user_id, room_id, content): """Add room to group diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py index 319565510f..ad22415782 100644 --- a/synapse/handlers/groups_local.py +++ b/synapse/handlers/groups_local.py @@ -63,7 +63,7 @@ def request_failed_errback(failure): return f -class GroupsLocalHandler(object): +class GroupsLocalWorkerHandler(object): def __init__(self, hs): self.hs = hs self.store = hs.get_datastore() @@ -81,40 +81,17 @@ def __init__(self, hs): self.profile_handler = hs.get_profile_handler() - # Ensure attestations get renewed - hs.get_groups_attestation_renewer() - # The following functions merely route the query to the local groups server # or federation depending on if the group is local or remote get_group_profile = _create_rerouter("get_group_profile") - update_group_profile = _create_rerouter("update_group_profile") get_rooms_in_group = _create_rerouter("get_rooms_in_group") - get_invited_users_in_group = _create_rerouter("get_invited_users_in_group") - - add_room_to_group = _create_rerouter("add_room_to_group") - update_room_in_group = _create_rerouter("update_room_in_group") - remove_room_from_group = _create_rerouter("remove_room_from_group") - - update_group_summary_room = _create_rerouter("update_group_summary_room") - delete_group_summary_room = _create_rerouter("delete_group_summary_room") - - update_group_category = _create_rerouter("update_group_category") - delete_group_category = _create_rerouter("delete_group_category") get_group_category = _create_rerouter("get_group_category") get_group_categories = _create_rerouter("get_group_categories") - - update_group_summary_user = _create_rerouter("update_group_summary_user") - delete_group_summary_user = _create_rerouter("delete_group_summary_user") - - update_group_role = _create_rerouter("update_group_role") - delete_group_role = _create_rerouter("delete_group_role") get_group_role = _create_rerouter("get_group_role") get_group_roles = _create_rerouter("get_group_roles") - set_group_join_policy = _create_rerouter("set_group_join_policy") - @defer.inlineCallbacks def get_group_summary(self, group_id, requester_user_id): """Get the group summary for a group. @@ -169,6 +146,144 @@ def get_group_summary(self, group_id, requester_user_id): return res + @defer.inlineCallbacks + def get_users_in_group(self, group_id, requester_user_id): + """Get users in a group + """ + if self.is_mine_id(group_id): + res = yield self.groups_server_handler.get_users_in_group( + group_id, requester_user_id + ) + return res + + group_server_name = get_domain_from_id(group_id) + + try: + res = yield self.transport_client.get_users_in_group( + get_domain_from_id(group_id), group_id, requester_user_id + ) + except HttpResponseException as e: + raise e.to_synapse_error() + except RequestSendFailed: + raise SynapseError(502, "Failed to contact group server") + + chunk = res["chunk"] + valid_entries = [] + for entry in chunk: + g_user_id = entry["user_id"] + attestation = entry.pop("attestation", {}) + try: + if get_domain_from_id(g_user_id) != group_server_name: + yield self.attestations.verify_attestation( + attestation, + group_id=group_id, + user_id=g_user_id, + server_name=get_domain_from_id(g_user_id), + ) + valid_entries.append(entry) + except Exception as e: + logger.info("Failed to verify user is in group: %s", e) + + res["chunk"] = valid_entries + + return res + + @defer.inlineCallbacks + def get_joined_groups(self, user_id): + group_ids = yield self.store.get_joined_groups(user_id) + return {"groups": group_ids} + + @defer.inlineCallbacks + def get_publicised_groups_for_user(self, user_id): + if self.hs.is_mine_id(user_id): + result = yield self.store.get_publicised_groups_for_user(user_id) + + # Check AS associated groups for this user - this depends on the + # RegExps in the AS registration file (under `users`) + for app_service in self.store.get_app_services(): + result.extend(app_service.get_groups_for_user(user_id)) + + return {"groups": result} + else: + try: + bulk_result = yield self.transport_client.bulk_get_publicised_groups( + get_domain_from_id(user_id), [user_id] + ) + except HttpResponseException as e: + raise e.to_synapse_error() + except RequestSendFailed: + raise SynapseError(502, "Failed to contact group server") + + result = bulk_result.get("users", {}).get(user_id) + # TODO: Verify attestations + return {"groups": result} + + @defer.inlineCallbacks + def bulk_get_publicised_groups(self, user_ids, proxy=True): + destinations = {} + local_users = set() + + for user_id in user_ids: + if self.hs.is_mine_id(user_id): + local_users.add(user_id) + else: + destinations.setdefault(get_domain_from_id(user_id), set()).add(user_id) + + if not proxy and destinations: + raise SynapseError(400, "Some user_ids are not local") + + results = {} + failed_results = [] + for destination, dest_user_ids in iteritems(destinations): + try: + r = yield self.transport_client.bulk_get_publicised_groups( + destination, list(dest_user_ids) + ) + results.update(r["users"]) + except Exception: + failed_results.extend(dest_user_ids) + + for uid in local_users: + results[uid] = yield self.store.get_publicised_groups_for_user(uid) + + # Check AS associated groups for this user - this depends on the + # RegExps in the AS registration file (under `users`) + for app_service in self.store.get_app_services(): + results[uid].extend(app_service.get_groups_for_user(uid)) + + return {"users": results} + + +class GroupsLocalHandler(GroupsLocalWorkerHandler): + def __init__(self, hs): + super(GroupsLocalHandler, self).__init__(hs) + + # Ensure attestations get renewed + hs.get_groups_attestation_renewer() + + # The following functions merely route the query to the local groups server + # or federation depending on if the group is local or remote + + update_group_profile = _create_rerouter("update_group_profile") + + add_room_to_group = _create_rerouter("add_room_to_group") + update_room_in_group = _create_rerouter("update_room_in_group") + remove_room_from_group = _create_rerouter("remove_room_from_group") + + update_group_summary_room = _create_rerouter("update_group_summary_room") + delete_group_summary_room = _create_rerouter("delete_group_summary_room") + + update_group_category = _create_rerouter("update_group_category") + delete_group_category = _create_rerouter("delete_group_category") + + update_group_summary_user = _create_rerouter("update_group_summary_user") + delete_group_summary_user = _create_rerouter("delete_group_summary_user") + + update_group_role = _create_rerouter("update_group_role") + delete_group_role = _create_rerouter("delete_group_role") + + set_group_join_policy = _create_rerouter("set_group_join_policy") + @defer.inlineCallbacks def create_group(self, group_id, user_id, content): """Create a group @@ -219,48 +334,6 @@ def create_group(self, group_id, user_id, content): return res - @defer.inlineCallbacks - def get_users_in_group(self, group_id, requester_user_id): - """Get users in a group - """ - if self.is_mine_id(group_id): - res = yield self.groups_server_handler.get_users_in_group( - group_id, requester_user_id - ) - return res - - group_server_name = get_domain_from_id(group_id) - - try: - res = yield self.transport_client.get_users_in_group( - get_domain_from_id(group_id), group_id, requester_user_id - ) - except HttpResponseException as e: - raise e.to_synapse_error() - except RequestSendFailed: - raise SynapseError(502, "Failed to contact group server") - - chunk = res["chunk"] - valid_entries = [] - for entry in chunk: - g_user_id = entry["user_id"] - attestation = entry.pop("attestation", {}) - try: - if get_domain_from_id(g_user_id) != group_server_name: - yield self.attestations.verify_attestation( - attestation, - group_id=group_id, - user_id=g_user_id, - server_name=get_domain_from_id(g_user_id), - ) - valid_entries.append(entry) - except Exception as e: - logger.info("Failed to verify user is in group: %s", e) - - res["chunk"] = valid_entries - - return res - @defer.inlineCallbacks def join_group(self, group_id, user_id, content): """Request to join a group @@ -452,68 +525,3 @@ def user_removed_from_group(self, group_id, user_id, content): group_id, user_id, membership="leave" ) self.notifier.on_new_event("groups_key", token, users=[user_id]) - - @defer.inlineCallbacks - def get_joined_groups(self, user_id): - group_ids = yield self.store.get_joined_groups(user_id) - return {"groups": group_ids} - - @defer.inlineCallbacks - def get_publicised_groups_for_user(self, user_id): - if self.hs.is_mine_id(user_id): - result = yield self.store.get_publicised_groups_for_user(user_id) - - # Check AS associated groups for this user - this depends on the - # RegExps in the AS registration file (under `users`) - for app_service in self.store.get_app_services(): - result.extend(app_service.get_groups_for_user(user_id)) - - return {"groups": result} - else: - try: - bulk_result = yield self.transport_client.bulk_get_publicised_groups( - get_domain_from_id(user_id), [user_id] - ) - except HttpResponseException as e: - raise e.to_synapse_error() - except RequestSendFailed: - raise SynapseError(502, "Failed to contact group server") - - result = bulk_result.get("users", {}).get(user_id) - # TODO: Verify attestations - return {"groups": result} - - @defer.inlineCallbacks - def bulk_get_publicised_groups(self, user_ids, proxy=True): - destinations = {} - local_users = set() - - for user_id in user_ids: - if self.hs.is_mine_id(user_id): - local_users.add(user_id) - else: - destinations.setdefault(get_domain_from_id(user_id), set()).add(user_id) - - if not proxy and destinations: - raise SynapseError(400, "Some user_ids are not local") - - results = {} - failed_results = [] - for destination, dest_user_ids in iteritems(destinations): - try: - r = yield self.transport_client.bulk_get_publicised_groups( - destination, list(dest_user_ids) - ) - results.update(r["users"]) - except Exception: - failed_results.extend(dest_user_ids) - - for uid in local_users: - results[uid] = yield self.store.get_publicised_groups_for_user(uid) - - # Check AS associated groups for this user - this depends on the - # RegExps in the AS registration file (under `users`) - for app_service in self.store.get_app_services(): - results[uid].extend(app_service.get_groups_for_user(uid)) - - return {"users": results} diff --git a/synapse/replication/slave/storage/groups.py b/synapse/replication/slave/storage/groups.py index 69a4ae42f9..2d4fd08cf5 100644 --- a/synapse/replication/slave/storage/groups.py +++ b/synapse/replication/slave/storage/groups.py @@ -13,15 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.storage import DataStore +from synapse.replication.slave.storage._base import BaseSlavedStore +from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker +from synapse.storage.data_stores.main.group_server import GroupServerWorkerStore from synapse.storage.database import Database from synapse.util.caches.stream_change_cache import StreamChangeCache -from ._base import BaseSlavedStore, __func__ -from ._slaved_id_tracker import SlavedIdTracker - -class SlavedGroupServerStore(BaseSlavedStore): +class SlavedGroupServerStore(GroupServerWorkerStore, BaseSlavedStore): def __init__(self, database: Database, db_conn, hs): super(SlavedGroupServerStore, self).__init__(database, db_conn, hs) @@ -35,9 +34,8 @@ def __init__(self, database: Database, db_conn, hs): self._group_updates_id_gen.get_current_token(), ) - get_groups_changes_for_user = __func__(DataStore.get_groups_changes_for_user) - get_group_stream_token = __func__(DataStore.get_group_stream_token) - get_all_groups_for_user = __func__(DataStore.get_all_groups_for_user) + def get_group_stream_token(self): + return self._group_updates_id_gen.get_current_token() def stream_positions(self): result = super(SlavedGroupServerStore, self).stream_positions() diff --git a/synapse/server.py b/synapse/server.py index 7926867b77..fd2f69e928 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -50,7 +50,7 @@ from synapse.federation.sender import FederationSender from synapse.federation.transport.client import TransportLayerClient from synapse.groups.attestations import GroupAttestationSigning, GroupAttestionRenewer -from synapse.groups.groups_server import GroupsServerHandler +from synapse.groups.groups_server import GroupsServerHandler, GroupsServerWorkerHandler from synapse.handlers import Handlers from synapse.handlers.account_validity import AccountValidityHandler from synapse.handlers.acme import AcmeHandler @@ -62,7 +62,7 @@ from synapse.handlers.e2e_keys import E2eKeysHandler from synapse.handlers.e2e_room_keys import E2eRoomKeysHandler from synapse.handlers.events import EventHandler, EventStreamHandler -from synapse.handlers.groups_local import GroupsLocalHandler +from synapse.handlers.groups_local import GroupsLocalHandler, GroupsLocalWorkerHandler from synapse.handlers.initial_sync import InitialSyncHandler from synapse.handlers.message import EventCreationHandler, MessageHandler from synapse.handlers.pagination import PaginationHandler @@ -460,10 +460,16 @@ def build_user_directory_handler(self): return UserDirectoryHandler(self) def build_groups_local_handler(self): - return GroupsLocalHandler(self) + if self.config.worker_app: + return GroupsLocalWorkerHandler(self) + else: + return GroupsLocalHandler(self) def build_groups_server_handler(self): - return GroupsServerHandler(self) + if self.config.worker_app: + return GroupsServerWorkerHandler(self) + else: + return GroupsServerHandler(self) def build_groups_attestation_signing(self): return GroupAttestationSigning(self) diff --git a/synapse/storage/data_stores/main/group_server.py b/synapse/storage/data_stores/main/group_server.py index 6acd45e9f3..0963e6c250 100644 --- a/synapse/storage/data_stores/main/group_server.py +++ b/synapse/storage/data_stores/main/group_server.py @@ -27,21 +27,7 @@ _DEFAULT_ROLE_ID = "" -class GroupServerStore(SQLBaseStore): - def set_group_join_policy(self, group_id, join_policy): - """Set the join policy of a group. - - join_policy can be one of: - * "invite" - * "open" - """ - return self.db.simple_update_one( - table="groups", - keyvalues={"group_id": group_id}, - updatevalues={"join_policy": join_policy}, - desc="set_group_join_policy", - ) - +class GroupServerWorkerStore(SQLBaseStore): def get_group(self, group_id): return self.db.simple_select_one( table="groups", @@ -157,6 +143,366 @@ def _get_rooms_for_summary_txn(txn): "get_rooms_for_summary", _get_rooms_for_summary_txn ) + @defer.inlineCallbacks + def get_group_categories(self, group_id): + rows = yield self.db.simple_select_list( + table="group_room_categories", + keyvalues={"group_id": group_id}, + retcols=("category_id", "is_public", "profile"), + desc="get_group_categories", + ) + + return { + row["category_id"]: { + "is_public": row["is_public"], + "profile": json.loads(row["profile"]), + } + for row in rows + } + + @defer.inlineCallbacks + def get_group_category(self, group_id, category_id): + category = yield self.db.simple_select_one( + table="group_room_categories", + keyvalues={"group_id": group_id, "category_id": category_id}, + retcols=("is_public", "profile"), + desc="get_group_category", + ) + + category["profile"] = json.loads(category["profile"]) + + return category + + @defer.inlineCallbacks + def get_group_roles(self, group_id): + rows = yield self.db.simple_select_list( + table="group_roles", + keyvalues={"group_id": group_id}, + retcols=("role_id", "is_public", "profile"), + desc="get_group_roles", + ) + + return { + row["role_id"]: { + "is_public": row["is_public"], + "profile": json.loads(row["profile"]), + } + for row in rows + } + + @defer.inlineCallbacks + def get_group_role(self, group_id, role_id): + role = yield self.db.simple_select_one( + table="group_roles", + keyvalues={"group_id": group_id, "role_id": role_id}, + retcols=("is_public", "profile"), + desc="get_group_role", + ) + + role["profile"] = json.loads(role["profile"]) + + return role + + def get_local_groups_for_room(self, room_id): + """Get all of the local group that contain a given room + Args: + room_id (str): The ID of a room + Returns: + Deferred[list[str]]: A twisted.Deferred containing a list of group ids + containing this room + """ + return self.db.simple_select_onecol( + table="group_rooms", + keyvalues={"room_id": room_id}, + retcol="group_id", + desc="get_local_groups_for_room", + ) + + def get_users_for_summary_by_role(self, group_id, include_private=False): + """Get the users and roles that should be included in a summary request + + Returns ([users], [roles]) + """ + + def _get_users_for_summary_txn(txn): + keyvalues = {"group_id": group_id} + if not include_private: + keyvalues["is_public"] = True + + sql = """ + SELECT user_id, is_public, role_id, user_order + FROM group_summary_users + WHERE group_id = ? + """ + + if not include_private: + sql += " AND is_public = ?" + txn.execute(sql, (group_id, True)) + else: + txn.execute(sql, (group_id,)) + + users = [ + { + "user_id": row[0], + "is_public": row[1], + "role_id": row[2] if row[2] != _DEFAULT_ROLE_ID else None, + "order": row[3], + } + for row in txn + ] + + sql = """ + SELECT role_id, is_public, profile, role_order + FROM group_summary_roles + INNER JOIN group_roles USING (group_id, role_id) + WHERE group_id = ? + """ + + if not include_private: + sql += " AND is_public = ?" + txn.execute(sql, (group_id, True)) + else: + txn.execute(sql, (group_id,)) + + roles = { + row[0]: { + "is_public": row[1], + "profile": json.loads(row[2]), + "order": row[3], + } + for row in txn + } + + return users, roles + + return self.db.runInteraction( + "get_users_for_summary_by_role", _get_users_for_summary_txn + ) + + def is_user_in_group(self, user_id, group_id): + return self.db.simple_select_one_onecol( + table="group_users", + keyvalues={"group_id": group_id, "user_id": user_id}, + retcol="user_id", + allow_none=True, + desc="is_user_in_group", + ).addCallback(lambda r: bool(r)) + + def is_user_admin_in_group(self, group_id, user_id): + return self.db.simple_select_one_onecol( + table="group_users", + keyvalues={"group_id": group_id, "user_id": user_id}, + retcol="is_admin", + allow_none=True, + desc="is_user_admin_in_group", + ) + + def is_user_invited_to_local_group(self, group_id, user_id): + """Has the group server invited a user? + """ + return self.db.simple_select_one_onecol( + table="group_invites", + keyvalues={"group_id": group_id, "user_id": user_id}, + retcol="user_id", + desc="is_user_invited_to_local_group", + allow_none=True, + ) + + def get_users_membership_info_in_group(self, group_id, user_id): + """Get a dict describing the membership of a user in a group. + + Example if joined: + + { + "membership": "join", + "is_public": True, + "is_privileged": False, + } + + Returns an empty dict if the user is not join/invite/etc + """ + + def _get_users_membership_in_group_txn(txn): + row = self.db.simple_select_one_txn( + txn, + table="group_users", + keyvalues={"group_id": group_id, "user_id": user_id}, + retcols=("is_admin", "is_public"), + allow_none=True, + ) + + if row: + return { + "membership": "join", + "is_public": row["is_public"], + "is_privileged": row["is_admin"], + } + + row = self.db.simple_select_one_onecol_txn( + txn, + table="group_invites", + keyvalues={"group_id": group_id, "user_id": user_id}, + retcol="user_id", + allow_none=True, + ) + + if row: + return {"membership": "invite"} + + return {} + + return self.db.runInteraction( + "get_users_membership_info_in_group", _get_users_membership_in_group_txn + ) + + def get_publicised_groups_for_user(self, user_id): + """Get all groups a user is publicising + """ + return self.db.simple_select_onecol( + table="local_group_membership", + keyvalues={"user_id": user_id, "membership": "join", "is_publicised": True}, + retcol="group_id", + desc="get_publicised_groups_for_user", + ) + + def get_attestations_need_renewals(self, valid_until_ms): + """Get all attestations that need to be renewed until givent time + """ + + def _get_attestations_need_renewals_txn(txn): + sql = """ + SELECT group_id, user_id FROM group_attestations_renewals + WHERE valid_until_ms <= ? + """ + txn.execute(sql, (valid_until_ms,)) + return self.db.cursor_to_dict(txn) + + return self.db.runInteraction( + "get_attestations_need_renewals", _get_attestations_need_renewals_txn + ) + + @defer.inlineCallbacks + def get_remote_attestation(self, group_id, user_id): + """Get the attestation that proves the remote agrees that the user is + in the group. + """ + row = yield self.db.simple_select_one( + table="group_attestations_remote", + keyvalues={"group_id": group_id, "user_id": user_id}, + retcols=("valid_until_ms", "attestation_json"), + desc="get_remote_attestation", + allow_none=True, + ) + + now = int(self._clock.time_msec()) + if row and now < row["valid_until_ms"]: + return json.loads(row["attestation_json"]) + + return None + + def get_joined_groups(self, user_id): + return self.db.simple_select_onecol( + table="local_group_membership", + keyvalues={"user_id": user_id, "membership": "join"}, + retcol="group_id", + desc="get_joined_groups", + ) + + def get_all_groups_for_user(self, user_id, now_token): + def _get_all_groups_for_user_txn(txn): + sql = """ + SELECT group_id, type, membership, u.content + FROM local_group_updates AS u + INNER JOIN local_group_membership USING (group_id, user_id) + WHERE user_id = ? AND membership != 'leave' + AND stream_id <= ? + """ + txn.execute(sql, (user_id, now_token)) + return [ + { + "group_id": row[0], + "type": row[1], + "membership": row[2], + "content": json.loads(row[3]), + } + for row in txn + ] + + return self.db.runInteraction( + "get_all_groups_for_user", _get_all_groups_for_user_txn + ) + + def get_groups_changes_for_user(self, user_id, from_token, to_token): + from_token = int(from_token) + has_changed = self._group_updates_stream_cache.has_entity_changed( + user_id, from_token + ) + if not has_changed: + return defer.succeed([]) + + def _get_groups_changes_for_user_txn(txn): + sql = """ + SELECT group_id, membership, type, u.content + FROM local_group_updates AS u + INNER JOIN local_group_membership USING (group_id, user_id) + WHERE user_id = ? AND ? < stream_id AND stream_id <= ? + """ + txn.execute(sql, (user_id, from_token, to_token)) + return [ + { + "group_id": group_id, + "membership": membership, + "type": gtype, + "content": json.loads(content_json), + } + for group_id, membership, gtype, content_json in txn + ] + + return self.db.runInteraction( + "get_groups_changes_for_user", _get_groups_changes_for_user_txn + ) + + def get_all_groups_changes(self, from_token, to_token, limit): + from_token = int(from_token) + has_changed = self._group_updates_stream_cache.has_any_entity_changed( + from_token + ) + if not has_changed: + return defer.succeed([]) + + def _get_all_groups_changes_txn(txn): + sql = """ + SELECT stream_id, group_id, user_id, type, content + FROM local_group_updates + WHERE ? < stream_id AND stream_id <= ? + LIMIT ? + """ + txn.execute(sql, (from_token, to_token, limit)) + return [ + (stream_id, group_id, user_id, gtype, json.loads(content_json)) + for stream_id, group_id, user_id, gtype, content_json in txn + ] + + return self.db.runInteraction( + "get_all_groups_changes", _get_all_groups_changes_txn + ) + + +class GroupServerStore(GroupServerWorkerStore): + def set_group_join_policy(self, group_id, join_policy): + """Set the join policy of a group. + + join_policy can be one of: + * "invite" + * "open" + """ + return self.db.simple_update_one( + table="groups", + keyvalues={"group_id": group_id}, + updatevalues={"join_policy": join_policy}, + desc="set_group_join_policy", + ) + def add_room_to_summary(self, group_id, room_id, category_id, order, is_public): return self.db.runInteraction( "add_room_to_summary", @@ -299,36 +645,6 @@ def remove_room_from_summary(self, group_id, room_id, category_id): desc="remove_room_from_summary", ) - @defer.inlineCallbacks - def get_group_categories(self, group_id): - rows = yield self.db.simple_select_list( - table="group_room_categories", - keyvalues={"group_id": group_id}, - retcols=("category_id", "is_public", "profile"), - desc="get_group_categories", - ) - - return { - row["category_id"]: { - "is_public": row["is_public"], - "profile": json.loads(row["profile"]), - } - for row in rows - } - - @defer.inlineCallbacks - def get_group_category(self, group_id, category_id): - category = yield self.db.simple_select_one( - table="group_room_categories", - keyvalues={"group_id": group_id, "category_id": category_id}, - retcols=("is_public", "profile"), - desc="get_group_category", - ) - - category["profile"] = json.loads(category["profile"]) - - return category - def upsert_group_category(self, group_id, category_id, profile, is_public): """Add/update room category for group """ @@ -360,36 +676,6 @@ def remove_group_category(self, group_id, category_id): desc="remove_group_category", ) - @defer.inlineCallbacks - def get_group_roles(self, group_id): - rows = yield self.db.simple_select_list( - table="group_roles", - keyvalues={"group_id": group_id}, - retcols=("role_id", "is_public", "profile"), - desc="get_group_roles", - ) - - return { - row["role_id"]: { - "is_public": row["is_public"], - "profile": json.loads(row["profile"]), - } - for row in rows - } - - @defer.inlineCallbacks - def get_group_role(self, group_id, role_id): - role = yield self.db.simple_select_one( - table="group_roles", - keyvalues={"group_id": group_id, "role_id": role_id}, - retcols=("is_public", "profile"), - desc="get_group_role", - ) - - role["profile"] = json.loads(role["profile"]) - - return role - def upsert_group_role(self, group_id, role_id, profile, is_public): """Add/remove user role """ @@ -469,251 +755,99 @@ def _add_user_to_summary_txn( if not role_exists: raise SynapseError(400, "Role doesn't exist") - # TODO: Check role is part of the summary already - role_exists = self.db.simple_select_one_onecol_txn( - txn, - table="group_summary_roles", - keyvalues={"group_id": group_id, "role_id": role_id}, - retcol="group_id", - allow_none=True, - ) - if not role_exists: - # If not, add it with an order larger than all others - txn.execute( - """ - INSERT INTO group_summary_roles - (group_id, role_id, role_order) - SELECT ?, ?, COALESCE(MAX(role_order), 0) + 1 - FROM group_summary_roles - WHERE group_id = ? AND role_id = ? - """, - (group_id, role_id, group_id, role_id), - ) - - existing = self.db.simple_select_one_txn( - txn, - table="group_summary_users", - keyvalues={"group_id": group_id, "user_id": user_id, "role_id": role_id}, - retcols=("user_order", "is_public"), - allow_none=True, - ) - - if order is not None: - # Shuffle other users orders that come after the given order - sql = """ - UPDATE group_summary_users SET user_order = user_order + 1 - WHERE group_id = ? AND role_id = ? AND user_order >= ? - """ - txn.execute(sql, (group_id, role_id, order)) - elif not existing: - sql = """ - SELECT COALESCE(MAX(user_order), 0) + 1 FROM group_summary_users - WHERE group_id = ? AND role_id = ? - """ - txn.execute(sql, (group_id, role_id)) - (order,) = txn.fetchone() - - if existing: - to_update = {} - if order is not None: - to_update["user_order"] = order - if is_public is not None: - to_update["is_public"] = is_public - self.db.simple_update_txn( - txn, - table="group_summary_users", - keyvalues={ - "group_id": group_id, - "role_id": role_id, - "user_id": user_id, - }, - values=to_update, - ) - else: - if is_public is None: - is_public = True - - self.db.simple_insert_txn( - txn, - table="group_summary_users", - values={ - "group_id": group_id, - "role_id": role_id, - "user_id": user_id, - "user_order": order, - "is_public": is_public, - }, - ) - - def remove_user_from_summary(self, group_id, user_id, role_id): - if role_id is None: - role_id = _DEFAULT_ROLE_ID - - return self.db.simple_delete( - table="group_summary_users", - keyvalues={"group_id": group_id, "role_id": role_id, "user_id": user_id}, - desc="remove_user_from_summary", - ) - - def get_local_groups_for_room(self, room_id): - """Get all of the local group that contain a given room - Args: - room_id (str): The ID of a room - Returns: - Deferred[list[str]]: A twisted.Deferred containing a list of group ids - containing this room - """ - return self.db.simple_select_onecol( - table="group_rooms", - keyvalues={"room_id": room_id}, - retcol="group_id", - desc="get_local_groups_for_room", - ) - - def get_users_for_summary_by_role(self, group_id, include_private=False): - """Get the users and roles that should be included in a summary request - - Returns ([users], [roles]) - """ - - def _get_users_for_summary_txn(txn): - keyvalues = {"group_id": group_id} - if not include_private: - keyvalues["is_public"] = True - - sql = """ - SELECT user_id, is_public, role_id, user_order - FROM group_summary_users - WHERE group_id = ? - """ - - if not include_private: - sql += " AND is_public = ?" - txn.execute(sql, (group_id, True)) - else: - txn.execute(sql, (group_id,)) - - users = [ - { - "user_id": row[0], - "is_public": row[1], - "role_id": row[2] if row[2] != _DEFAULT_ROLE_ID else None, - "order": row[3], - } - for row in txn - ] - - sql = """ - SELECT role_id, is_public, profile, role_order - FROM group_summary_roles - INNER JOIN group_roles USING (group_id, role_id) - WHERE group_id = ? - """ - - if not include_private: - sql += " AND is_public = ?" - txn.execute(sql, (group_id, True)) - else: - txn.execute(sql, (group_id,)) - - roles = { - row[0]: { - "is_public": row[1], - "profile": json.loads(row[2]), - "order": row[3], - } - for row in txn - } - - return users, roles - - return self.db.runInteraction( - "get_users_for_summary_by_role", _get_users_for_summary_txn - ) - - def is_user_in_group(self, user_id, group_id): - return self.db.simple_select_one_onecol( - table="group_users", - keyvalues={"group_id": group_id, "user_id": user_id}, - retcol="user_id", - allow_none=True, - desc="is_user_in_group", - ).addCallback(lambda r: bool(r)) - - def is_user_admin_in_group(self, group_id, user_id): - return self.db.simple_select_one_onecol( - table="group_users", - keyvalues={"group_id": group_id, "user_id": user_id}, - retcol="is_admin", - allow_none=True, - desc="is_user_admin_in_group", - ) - - def add_group_invite(self, group_id, user_id): - """Record that the group server has invited a user - """ - return self.db.simple_insert( - table="group_invites", - values={"group_id": group_id, "user_id": user_id}, - desc="add_group_invite", - ) + # TODO: Check role is part of the summary already + role_exists = self.db.simple_select_one_onecol_txn( + txn, + table="group_summary_roles", + keyvalues={"group_id": group_id, "role_id": role_id}, + retcol="group_id", + allow_none=True, + ) + if not role_exists: + # If not, add it with an order larger than all others + txn.execute( + """ + INSERT INTO group_summary_roles + (group_id, role_id, role_order) + SELECT ?, ?, COALESCE(MAX(role_order), 0) + 1 + FROM group_summary_roles + WHERE group_id = ? AND role_id = ? + """, + (group_id, role_id, group_id, role_id), + ) - def is_user_invited_to_local_group(self, group_id, user_id): - """Has the group server invited a user? - """ - return self.db.simple_select_one_onecol( - table="group_invites", - keyvalues={"group_id": group_id, "user_id": user_id}, - retcol="user_id", - desc="is_user_invited_to_local_group", + existing = self.db.simple_select_one_txn( + txn, + table="group_summary_users", + keyvalues={"group_id": group_id, "user_id": user_id, "role_id": role_id}, + retcols=("user_order", "is_public"), allow_none=True, ) - def get_users_membership_info_in_group(self, group_id, user_id): - """Get a dict describing the membership of a user in a group. - - Example if joined: - - { - "membership": "join", - "is_public": True, - "is_privileged": False, - } - - Returns an empty dict if the user is not join/invite/etc - """ + if order is not None: + # Shuffle other users orders that come after the given order + sql = """ + UPDATE group_summary_users SET user_order = user_order + 1 + WHERE group_id = ? AND role_id = ? AND user_order >= ? + """ + txn.execute(sql, (group_id, role_id, order)) + elif not existing: + sql = """ + SELECT COALESCE(MAX(user_order), 0) + 1 FROM group_summary_users + WHERE group_id = ? AND role_id = ? + """ + txn.execute(sql, (group_id, role_id)) + (order,) = txn.fetchone() - def _get_users_membership_in_group_txn(txn): - row = self.db.simple_select_one_txn( + if existing: + to_update = {} + if order is not None: + to_update["user_order"] = order + if is_public is not None: + to_update["is_public"] = is_public + self.db.simple_update_txn( txn, - table="group_users", - keyvalues={"group_id": group_id, "user_id": user_id}, - retcols=("is_admin", "is_public"), - allow_none=True, + table="group_summary_users", + keyvalues={ + "group_id": group_id, + "role_id": role_id, + "user_id": user_id, + }, + values=to_update, ) + else: + if is_public is None: + is_public = True - if row: - return { - "membership": "join", - "is_public": row["is_public"], - "is_privileged": row["is_admin"], - } - - row = self.db.simple_select_one_onecol_txn( + self.db.simple_insert_txn( txn, - table="group_invites", - keyvalues={"group_id": group_id, "user_id": user_id}, - retcol="user_id", - allow_none=True, + table="group_summary_users", + values={ + "group_id": group_id, + "role_id": role_id, + "user_id": user_id, + "user_order": order, + "is_public": is_public, + }, ) - if row: - return {"membership": "invite"} + def remove_user_from_summary(self, group_id, user_id, role_id): + if role_id is None: + role_id = _DEFAULT_ROLE_ID - return {} + return self.db.simple_delete( + table="group_summary_users", + keyvalues={"group_id": group_id, "role_id": role_id, "user_id": user_id}, + desc="remove_user_from_summary", + ) - return self.db.runInteraction( - "get_users_membership_info_in_group", _get_users_membership_in_group_txn + def add_group_invite(self, group_id, user_id): + """Record that the group server has invited a user + """ + return self.db.simple_insert( + table="group_invites", + values={"group_id": group_id, "user_id": user_id}, + desc="add_group_invite", ) def add_user_to_group( @@ -846,16 +980,6 @@ def _remove_room_from_group_txn(txn): "remove_room_from_group", _remove_room_from_group_txn ) - def get_publicised_groups_for_user(self, user_id): - """Get all groups a user is publicising - """ - return self.db.simple_select_onecol( - table="local_group_membership", - keyvalues={"user_id": user_id, "membership": "join", "is_publicised": True}, - retcol="group_id", - desc="get_publicised_groups_for_user", - ) - def update_group_publicity(self, group_id, user_id, publicise): """Update whether the user is publicising their membership of the group """ @@ -1000,22 +1124,6 @@ def update_group_profile(self, group_id, profile): desc="update_group_profile", ) - def get_attestations_need_renewals(self, valid_until_ms): - """Get all attestations that need to be renewed until givent time - """ - - def _get_attestations_need_renewals_txn(txn): - sql = """ - SELECT group_id, user_id FROM group_attestations_renewals - WHERE valid_until_ms <= ? - """ - txn.execute(sql, (valid_until_ms,)) - return self.db.cursor_to_dict(txn) - - return self.db.runInteraction( - "get_attestations_need_renewals", _get_attestations_need_renewals_txn - ) - def update_attestation_renewal(self, group_id, user_id, attestation): """Update an attestation that we have renewed """ @@ -1054,112 +1162,6 @@ def remove_attestation_renewal(self, group_id, user_id): desc="remove_attestation_renewal", ) - @defer.inlineCallbacks - def get_remote_attestation(self, group_id, user_id): - """Get the attestation that proves the remote agrees that the user is - in the group. - """ - row = yield self.db.simple_select_one( - table="group_attestations_remote", - keyvalues={"group_id": group_id, "user_id": user_id}, - retcols=("valid_until_ms", "attestation_json"), - desc="get_remote_attestation", - allow_none=True, - ) - - now = int(self._clock.time_msec()) - if row and now < row["valid_until_ms"]: - return json.loads(row["attestation_json"]) - - return None - - def get_joined_groups(self, user_id): - return self.db.simple_select_onecol( - table="local_group_membership", - keyvalues={"user_id": user_id, "membership": "join"}, - retcol="group_id", - desc="get_joined_groups", - ) - - def get_all_groups_for_user(self, user_id, now_token): - def _get_all_groups_for_user_txn(txn): - sql = """ - SELECT group_id, type, membership, u.content - FROM local_group_updates AS u - INNER JOIN local_group_membership USING (group_id, user_id) - WHERE user_id = ? AND membership != 'leave' - AND stream_id <= ? - """ - txn.execute(sql, (user_id, now_token)) - return [ - { - "group_id": row[0], - "type": row[1], - "membership": row[2], - "content": json.loads(row[3]), - } - for row in txn - ] - - return self.db.runInteraction( - "get_all_groups_for_user", _get_all_groups_for_user_txn - ) - - def get_groups_changes_for_user(self, user_id, from_token, to_token): - from_token = int(from_token) - has_changed = self._group_updates_stream_cache.has_entity_changed( - user_id, from_token - ) - if not has_changed: - return defer.succeed([]) - - def _get_groups_changes_for_user_txn(txn): - sql = """ - SELECT group_id, membership, type, u.content - FROM local_group_updates AS u - INNER JOIN local_group_membership USING (group_id, user_id) - WHERE user_id = ? AND ? < stream_id AND stream_id <= ? - """ - txn.execute(sql, (user_id, from_token, to_token)) - return [ - { - "group_id": group_id, - "membership": membership, - "type": gtype, - "content": json.loads(content_json), - } - for group_id, membership, gtype, content_json in txn - ] - - return self.db.runInteraction( - "get_groups_changes_for_user", _get_groups_changes_for_user_txn - ) - - def get_all_groups_changes(self, from_token, to_token, limit): - from_token = int(from_token) - has_changed = self._group_updates_stream_cache.has_any_entity_changed( - from_token - ) - if not has_changed: - return defer.succeed([]) - - def _get_all_groups_changes_txn(txn): - sql = """ - SELECT stream_id, group_id, user_id, type, content - FROM local_group_updates - WHERE ? < stream_id AND stream_id <= ? - LIMIT ? - """ - txn.execute(sql, (from_token, to_token, limit)) - return [ - (stream_id, group_id, user_id, gtype, json.loads(content_json)) - for stream_id, group_id, user_id, gtype, content_json in txn - ] - - return self.db.runInteraction( - "get_all_groups_changes", _get_all_groups_changes_txn - ) - def get_group_stream_token(self): return self._group_updates_id_gen.get_current_token() From 8a29def84abb8e71b1e75746ead8394fddc5843f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 Feb 2020 15:45:39 +0000 Subject: [PATCH 094/178] Add support for putting fed user query API on workers (#6873) --- changelog.d/6873.feature | 1 + docs/workers.md | 1 + synapse/app/federation_reader.py | 2 ++ synapse/federation/federation_server.py | 7 +++-- synapse/handlers/device.py | 35 +++++++++++-------------- 5 files changed, 25 insertions(+), 21 deletions(-) create mode 100644 changelog.d/6873.feature diff --git a/changelog.d/6873.feature b/changelog.d/6873.feature new file mode 100644 index 0000000000..bbedf8f7f0 --- /dev/null +++ b/changelog.d/6873.feature @@ -0,0 +1 @@ +Add ability to route federation user device queries to workers. diff --git a/docs/workers.md b/docs/workers.md index 82442d6a0a..6f7ec58780 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -176,6 +176,7 @@ endpoints matching the following regular expressions: ^/_matrix/federation/v1/query_auth/ ^/_matrix/federation/v1/event_auth/ ^/_matrix/federation/v1/exchange_third_party_invite/ + ^/_matrix/federation/v1/user/devices/ ^/_matrix/federation/v1/send/ ^/_matrix/federation/v1/get_groups_publicised$ ^/_matrix/key/v2/query diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 5e17ef1396..d055d11b23 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -33,6 +33,7 @@ from synapse.replication.slave.storage._base import BaseSlavedStore from synapse.replication.slave.storage.account_data import SlavedAccountDataStore from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore +from synapse.replication.slave.storage.devices import SlavedDeviceStore from synapse.replication.slave.storage.directory import DirectoryStore from synapse.replication.slave.storage.events import SlavedEventStore from synapse.replication.slave.storage.groups import SlavedGroupServerStore @@ -68,6 +69,7 @@ class FederationReaderSlavedStore( SlavedKeyStore, SlavedRegistrationStore, SlavedGroupServerStore, + SlavedDeviceStore, RoomStore, DirectoryStore, SlavedTransactionStore, diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index a4c97ed458..b3e4db507e 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -82,6 +82,8 @@ def __init__(self, hs): self.handler = hs.get_handlers().federation_handler self.state = hs.get_state_handler() + self.device_handler = hs.get_device_handler() + self._server_linearizer = Linearizer("fed_server") self._transaction_linearizer = Linearizer("fed_txn_handler") @@ -528,8 +530,9 @@ async def on_query_auth_request(self, origin, content, room_id, event_id): def on_query_client_keys(self, origin, content): return self.on_query_request("client_keys", content) - def on_query_user_devices(self, origin, user_id): - return self.on_query_request("user_devices", user_id) + async def on_query_user_devices(self, origin: str, user_id: str): + keys = await self.device_handler.on_federation_query_user_devices(user_id) + return 200, keys @trace async def on_claim_client_keys(self, origin, content): diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index a9bd431486..6d8e48ed39 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -225,6 +225,22 @@ def get_user_ids_changed(self, user_id, from_token): return result + @defer.inlineCallbacks + def on_federation_query_user_devices(self, user_id): + stream_id, devices = yield self.store.get_devices_with_keys_by_user(user_id) + master_key = yield self.store.get_e2e_cross_signing_key(user_id, "master") + self_signing_key = yield self.store.get_e2e_cross_signing_key( + user_id, "self_signing" + ) + + return { + "user_id": user_id, + "stream_id": stream_id, + "devices": devices, + "master_key": master_key, + "self_signing_key": self_signing_key, + } + class DeviceHandler(DeviceWorkerHandler): def __init__(self, hs): @@ -239,9 +255,6 @@ def __init__(self, hs): federation_registry.register_edu_handler( "m.device_list_update", self.device_list_updater.incoming_device_list_update ) - federation_registry.register_query_handler( - "user_devices", self.on_federation_query_user_devices - ) hs.get_distributor().observe("user_left_room", self.user_left_room) @@ -456,22 +469,6 @@ def notify_user_signature_update(self, from_user_id, user_ids): self.notifier.on_new_event("device_list_key", position, users=[from_user_id]) - @defer.inlineCallbacks - def on_federation_query_user_devices(self, user_id): - stream_id, devices = yield self.store.get_devices_with_keys_by_user(user_id) - master_key = yield self.store.get_e2e_cross_signing_key(user_id, "master") - self_signing_key = yield self.store.get_e2e_cross_signing_key( - user_id, "self_signing" - ) - - return { - "user_id": user_id, - "stream_id": stream_id, - "devices": devices, - "master_key": master_key, - "self_signing_key": self_signing_key, - } - @defer.inlineCallbacks def user_left_room(self, user, room_id): user_id = user.to_string() From d840ee5bdefcd4278a65a8f38d50ad9c9cace677 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 27 Feb 2020 10:44:55 +0000 Subject: [PATCH 095/178] Revert "skip send without trailing slash" I think this was done back when most synapses would reject the no-trailing-slash version; it's no longer required, and makes matrix.org spec-incompliant. This reverts commit fc5be50d561dcf0f069c5b4920faa0dfd3a962e2. --- synapse/federation/transport/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 9c6f22760d..dc563538de 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -152,7 +152,7 @@ def send_transaction(self, transaction, json_data_callback=None): # generated by the json_data_callback. json_data = transaction.get_dict() - path = _create_v1_path("/send/%s/", transaction.transaction_id) + path = _create_v1_path("/send/%s", transaction.transaction_id) response = yield self.client.put_json( transaction.destination, @@ -161,7 +161,7 @@ def send_transaction(self, transaction, json_data_callback=None): json_data_callback=json_data_callback, long_retries=True, backoff_on_404=True, # If we get a 404 the other side has gone - # try_trailing_slash_on_400=True, + try_trailing_slash_on_400=True, ) return response From 936686ed2dc5d5010cc4e58663b181c5ee0dcc5b Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Wed, 11 Mar 2020 15:21:25 +0000 Subject: [PATCH 096/178] Don't filter out events when we're checking the visibility of state --- synapse/handlers/message.py | 2 +- synapse/res/templates/saml_error.html | 42 +++++++++++++++++++++++++++ synapse/visibility.py | 15 +++++----- 3 files changed, 50 insertions(+), 9 deletions(-) create mode 100644 synapse/res/templates/saml_error.html diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 58faeab247..857bb478c2 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -160,7 +160,7 @@ def get_state_events( raise NotFoundError("Can't find event for token %s" % (at_token,)) visible_events = yield filter_events_for_client( - self.storage, user_id, last_events, apply_retention_policies=False + self.storage, user_id, last_events, filter_send_to_client=False ) event = last_events[0] diff --git a/synapse/res/templates/saml_error.html b/synapse/res/templates/saml_error.html new file mode 100644 index 0000000000..31841c3137 --- /dev/null +++ b/synapse/res/templates/saml_error.html @@ -0,0 +1,42 @@ + + + + + SSO error + + +

Oops! Something went wrong during authentication.

+

+ If you're seeing this page after clicking a link sent to you via email, make + sure you only click the confirmation link once, and that you open the + validation link in the same client you're logging in from. +

+

+ Try logging in again from your Matrix client and if the problem persists + please contact the server's administrator. +

+ + + + \ No newline at end of file diff --git a/synapse/visibility.py b/synapse/visibility.py index a48a4f3dfe..1d538b206d 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -49,7 +49,7 @@ def filter_events_for_client( events, is_peeking=False, always_include_ids=frozenset(), - apply_retention_policies=True, + filter_send_to_client=True, ): """ Check which events a user is allowed to see. If the user can see the event but its @@ -65,10 +65,9 @@ def filter_events_for_client( events always_include_ids (set(event_id)): set of event ids to specifically include (unless sender is ignored) - apply_retention_policies (bool): Whether to filter out events that's older than - allowed by the room's retention policy. Useful when this function is called - to e.g. check whether a user should be allowed to see the state at a given - event rather than to know if it should send an event to a user's client(s). + filter_send_to_client (bool): Whether we're checking an event that's going to be + sent to a client. This might not always be the case since this function can + also be called to check whether a user can see the state at a given point. Returns: Deferred[list[synapse.events.EventBase]] @@ -96,7 +95,7 @@ def filter_events_for_client( erased_senders = yield storage.main.are_users_erased((e.sender for e in events)) - if apply_retention_policies: + if not filter_send_to_client: room_ids = {e.room_id for e in events} retention_policies = {} @@ -119,7 +118,7 @@ def allowed(event): the original event if they can see it as normal. """ - if event.type == "org.matrix.dummy_event": + if event.type == "org.matrix.dummy_event" and filter_send_to_client: return None if not event.is_state() and event.sender in ignore_list: @@ -134,7 +133,7 @@ def allowed(event): # Don't try to apply the room's retention policy if the event is a state event, as # MSC1763 states that retention is only considered for non-state events. - if apply_retention_policies and not event.is_state(): + if filter_send_to_client and not event.is_state(): retention_policy = retention_policies[event.room_id] max_lifetime = retention_policy.get("max_lifetime") From 6b73b8b70c7b673987897d8e3f871f90c9c15d11 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Wed, 11 Mar 2020 15:32:07 +0000 Subject: [PATCH 097/178] Fix condition --- synapse/visibility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/visibility.py b/synapse/visibility.py index 1d538b206d..d0b2241e48 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -95,7 +95,7 @@ def filter_events_for_client( erased_senders = yield storage.main.are_users_erased((e.sender for e in events)) - if not filter_send_to_client: + if filter_send_to_client: room_ids = {e.room_id for e in events} retention_policies = {} From 568461b5ec3b4eebe68e6bbe21fc792c71af6a65 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Wed, 11 Mar 2020 17:04:18 +0000 Subject: [PATCH 098/178] Also don't filter out events sent by ignored users when checking state visibility --- synapse/visibility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/visibility.py b/synapse/visibility.py index d0b2241e48..82a2132427 100644 --- a/synapse/visibility.py +++ b/synapse/visibility.py @@ -121,7 +121,7 @@ def allowed(event): if event.type == "org.matrix.dummy_event" and filter_send_to_client: return None - if not event.is_state() and event.sender in ignore_list: + if not event.is_state() and event.sender in ignore_list and filter_send_to_client: return None # Until MSC2261 has landed we can't redact malicious alias events, so for From 08d68c52960ac3c2a36e1aaab0fd02d1f7fc324a Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 12 Mar 2020 15:59:24 +0000 Subject: [PATCH 099/178] Populate the room version from state events See `rooms_version_column_3.sql.postgres` for details about why we need to do that. --- .../57/rooms_version_column_3.sql.postgres | 38 +++++++++++++++++++ .../57/rooms_version_column_3.sql.sqlite | 22 +++++++++++ 2 files changed, 60 insertions(+) create mode 100644 synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres create mode 100644 synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite diff --git a/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres new file mode 100644 index 0000000000..31be4d8e73 --- /dev/null +++ b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres @@ -0,0 +1,38 @@ +/* Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- When we first added the room_version column to the rooms table, it was populated from +-- the current_state_events table. However, there was an issue causing a background +-- update to clean up the current_state_events table for rooms where the server is no +-- longer participating, before that column could be populated. Therefore, some rooms had +-- a NULL room_version. + +-- The rooms_version_column_2.sql.* delta files were introduced to make the populating +-- synchronous instead of running it in a background update, which fixed this issue. +-- However, all of the instances of Synapse installed or updated in the meantime got +-- their rooms table corrupted with NULL room_versions. + +-- This query fishes out the room versions from the create event using the state_events +-- table instead of the current_state_events one, as the former still have all of the +-- create events. + +UPDATE rooms SET room_version=( + SELECT COALESCE(json::json->'content'->>'room_version','1') + FROM state_events se INNER JOIN event_json ej USING (event_id) + WHERE se.room_id=rooms.room_id AND se.type='m.room.create' AND se.state_key='' +) WHERE rooms.room_version IS NULL; + +-- see also rooms_version_column_3.sql.sqlite which has a copy of the above query, using +-- sqlite syntax for the json extraction. diff --git a/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite new file mode 100644 index 0000000000..d13911a64e --- /dev/null +++ b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite @@ -0,0 +1,22 @@ +/* Copyright 2020 The Matrix.org Foundation C.I.C. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- see rooms_version_column_3.sql.postgres for details of what's going on here. + +UPDATE rooms SET room_version=( + SELECT COALESCE(json_extract(ej.json, '$.content.room_version'), '1') + FROM state_events se INNER JOIN event_json ej USING (event_id) + WHERE se.room_id=rooms.room_id AND se.type='m.room.create' AND se.state_key='' +) WHERE rooms.room_version IS NULL; From 03c694bb0885234618ff2762eade4a05f880f735 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 12 Mar 2020 16:48:11 +0000 Subject: [PATCH 100/178] Fix schema deltas --- .../main/schema/delta/57/rooms_version_column_3.sql.postgres | 1 + .../main/schema/delta/57/rooms_version_column_3.sql.sqlite | 1 + 2 files changed, 2 insertions(+) diff --git a/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres index 31be4d8e73..92aaadde0d 100644 --- a/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres +++ b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.postgres @@ -32,6 +32,7 @@ UPDATE rooms SET room_version=( SELECT COALESCE(json::json->'content'->>'room_version','1') FROM state_events se INNER JOIN event_json ej USING (event_id) WHERE se.room_id=rooms.room_id AND se.type='m.room.create' AND se.state_key='' + LIMIT 1 ) WHERE rooms.room_version IS NULL; -- see also rooms_version_column_3.sql.sqlite which has a copy of the above query, using diff --git a/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite index d13911a64e..e19dab97cb 100644 --- a/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite +++ b/synapse/storage/data_stores/main/schema/delta/57/rooms_version_column_3.sql.sqlite @@ -19,4 +19,5 @@ UPDATE rooms SET room_version=( SELECT COALESCE(json_extract(ej.json, '$.content.room_version'), '1') FROM state_events se INNER JOIN event_json ej USING (event_id) WHERE se.room_id=rooms.room_id AND se.type='m.room.create' AND se.state_key='' + LIMIT 1 ) WHERE rooms.room_version IS NULL; From 9b8212d2561c9d26b8cac2f46a8e4c76e2cc3994 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 17 Apr 2020 17:36:24 +0100 Subject: [PATCH 101/178] Update changelog --- changelog.d/7289.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/7289.bugfix b/changelog.d/7289.bugfix index 5b4fbd77ac..1568e1569b 100644 --- a/changelog.d/7289.bugfix +++ b/changelog.d/7289.bugfix @@ -1 +1 @@ -Fix an edge-case where it was not possible to cross-sign a user which did not share a room with any user on your homeserver. The bug only affected Synapse deployments in worker mode. +Fix a bug with cross-signing devices of users on other homeservers while in worker mode. \ No newline at end of file From fb3f1fb5c0192f7c4e29acdc987720a49f8f2f18 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 17 Apr 2020 17:36:53 +0100 Subject: [PATCH 102/178] Fix log lines, return type, tuple handling --- synapse/handlers/e2e_keys.py | 40 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index afc173ab2f..aa08e5bb97 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -987,12 +987,22 @@ def _get_e2e_cross_signing_verify_key( SynapseError: if `user_id` is invalid """ user = UserID.from_string(user_id) + key_id = None verify_key = None - key = yield self.store.get_e2e_cross_signing_key( user_id, key_type, from_user_id ) + if key is not None: + try: + key_id, verify_key = get_verify_key_from_cross_signing_key(key) + except ValueError as e: + logger.warning( + "Invalid %s key retrieved: %s - %s %s", key_type, key, type(e), e, + ) + raise SynapseError( + 502, "Invalid %s key retrieved from database" % (key_type,) + ) # If we couldn't find the key locally, and we're looking for keys of # another user then attempt to fetch the missing key from the remote @@ -1008,7 +1018,7 @@ def _get_e2e_cross_signing_verify_key( # We only get "master" and "self_signing" keys from remote servers and key_type in ["master", "self_signing"] ): - key = yield self._retrieve_cross_signing_keys_for_remote_user( + key, key_id, verify_key = yield self._retrieve_cross_signing_keys_for_remote_user( user, key_type ) @@ -1016,24 +1026,12 @@ def _get_e2e_cross_signing_verify_key( logger.debug("No %s key found for %s", key_type, user_id) raise NotFoundError("No %s key found for %s" % (key_type, user_id)) - # If we retrieved the keys remotely, these values will already be set - if key_id is None or verify_key is None: - try: - key_id, verify_key = get_verify_key_from_cross_signing_key(key) - except ValueError as e: - logger.debug( - "Invalid %s key retrieved: %s - %s %s", key_type, key, type(e), e, - ) - raise SynapseError( - 502, "Invalid %s key retrieved from remote server", key_type - ) - return key, key_id, verify_key @defer.inlineCallbacks def _retrieve_cross_signing_keys_for_remote_user( self, user: UserID, desired_key_type: str, - ) -> Tuple[Optional[Dict], Optional[str], Optional[VerifyKey]]: + ): """Queries cross-signing keys for a remote user and saves them to the database Only the key specified by `key_type` will be returned, while all retrieved keys @@ -1044,7 +1042,8 @@ def _retrieve_cross_signing_keys_for_remote_user( desired_key_type: The type of key to receive. One of "master", "self_signing" Returns: - A tuple of the retrieved key content, the key's ID and the matching VerifyKey. + Deferred[Tuple[Optional[Dict], Optional[str], Optional[VerifyKey]]]: A tuple + of the retrieved key content, the key's ID and the matching VerifyKey. If the key cannot be retrieved, all values in the tuple will instead be None. """ try: @@ -1059,7 +1058,7 @@ def _retrieve_cross_signing_keys_for_remote_user( type(e), e, ) - return None + return None, None, None # Process each of the retrieved cross-signing keys final_key = None @@ -1084,8 +1083,9 @@ def _retrieve_cross_signing_keys_for_remote_user( # algorithm and colon, which is the device ID key_id, verify_key = get_verify_key_from_cross_signing_key(key_content) except ValueError as e: - logger.debug( - "Invalid %s key retrieved: %s - %s %s", + logger.warning( + "Invalid %s key retrieved from remote %s: %s - %s %s", + user.domain, key_type, key_content, type(e), @@ -1094,7 +1094,7 @@ def _retrieve_cross_signing_keys_for_remote_user( continue device_ids.append(verify_key.version) - # If this is the desired key type, save it and it's ID/VerifyKey + # If this is the desired key type, save it and its ID/VerifyKey if key_type == desired_key_type: final_key = key_content final_verify_key = verify_key From 2fdfa96ee63b9838e1b0e255cae531687a726aca Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 17 Apr 2020 17:38:36 +0100 Subject: [PATCH 103/178] lint --- synapse/handlers/e2e_keys.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index aa08e5bb97..f096ea8a53 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -16,7 +16,6 @@ # limitations under the License. import logging -from typing import Dict, Optional, Tuple from six import iteritems @@ -24,7 +23,6 @@ from canonicaljson import encode_canonical_json, json from signedjson.key import decode_verify_key_bytes from signedjson.sign import SignatureVerifyException, verify_signed_json -from signedjson.types import VerifyKey from unpaddedbase64 import decode_base64 from twisted.internet import defer @@ -1018,9 +1016,11 @@ def _get_e2e_cross_signing_verify_key( # We only get "master" and "self_signing" keys from remote servers and key_type in ["master", "self_signing"] ): - key, key_id, verify_key = yield self._retrieve_cross_signing_keys_for_remote_user( - user, key_type - ) + ( + key, + key_id, + verify_key, + ) = yield self._retrieve_cross_signing_keys_for_remote_user(user, key_type) if key is None: logger.debug("No %s key found for %s", key_type, user_id) From da5e6eea450a22c540f3da1709f8b23c89f39999 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Apr 2020 11:00:57 +0100 Subject: [PATCH 104/178] Revert recent merges of #7289 into matrix-org-hotfixes This was incorrectly merged before it was ready. This reverts commit aead826d2dee183bc6003a22612911e1664246c3, reversing changes made to 4cd2a4ae3a0722816371766b31515432ec7ada6c. It also reverts commits 9b8212d25, fb3f1fb5c and 2fdfa96ee. --- changelog.d/7289.bugfix | 1 - synapse/federation/transport/client.py | 14 +-- synapse/handlers/e2e_keys.py | 138 ++----------------------- 3 files changed, 12 insertions(+), 141 deletions(-) delete mode 100644 changelog.d/7289.bugfix diff --git a/changelog.d/7289.bugfix b/changelog.d/7289.bugfix deleted file mode 100644 index 1568e1569b..0000000000 --- a/changelog.d/7289.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug with cross-signing devices of users on other homeservers while in worker mode. \ No newline at end of file diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index c35637a571..dc563538de 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -406,19 +406,13 @@ def query_client_keys(self, destination, query_content, timeout): "device_keys": { "": { "": {...} - } } - "master_keys": { - "": {...} - } } - "self_signing_keys": { - "": {...} } } } Args: destination(str): The server to query. query_content(dict): The user ids to query. Returns: - A dict containing device and cross-signing keys. + A dict containg the device keys. """ path = _create_v1_path("/user/keys/query") @@ -435,16 +429,14 @@ def query_user_devices(self, destination, user_id, timeout): Response: { "stream_id": "...", - "devices": [ { ... } ], - "master_key": { ... }, - "self_signing_key: { ... } + "devices": [ { ... } ] } Args: destination(str): The server to query. query_content(dict): The user ids to query. Returns: - A dict containing device and cross-signing keys. + A dict containg the device keys. """ path = _create_v1_path("/user/devices/%s", user_id) diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index f096ea8a53..8d7075f2eb 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -174,8 +174,8 @@ def do_remote_query(destination): """This is called when we are querying the device list of a user on a remote homeserver and their device list is not in the device list cache. If we share a room with this user and we're not querying for - specific user we will update the cache with their device list. - """ + specific user we will update the cache + with their device list.""" destination_query = remote_queries_not_in_cache[destination] @@ -961,19 +961,13 @@ def _process_other_signatures(self, user_id, signatures): return signature_list, failures @defer.inlineCallbacks - def _get_e2e_cross_signing_verify_key( - self, user_id: str, key_type: str, from_user_id: str = None - ): - """Fetch locally or remotely query for a cross-signing public key. - - First, attempt to fetch the cross-signing public key from storage. - If that fails, query the keys from the homeserver they belong to - and update our local copy. + def _get_e2e_cross_signing_verify_key(self, user_id, key_type, from_user_id=None): + """Fetch the cross-signing public key from storage and interpret it. Args: - user_id: the user whose key should be fetched - key_type: the type of key to fetch - from_user_id: the user that we are fetching the keys for. + user_id (str): the user whose key should be fetched + key_type (str): the type of key to fetch + from_user_id (str): the user that we are fetching the keys for. This affects what signatures are fetched. Returns: @@ -982,130 +976,16 @@ def _get_e2e_cross_signing_verify_key( Raises: NotFoundError: if the key is not found - SynapseError: if `user_id` is invalid """ - user = UserID.from_string(user_id) - - key_id = None - verify_key = None key = yield self.store.get_e2e_cross_signing_key( user_id, key_type, from_user_id ) - if key is not None: - try: - key_id, verify_key = get_verify_key_from_cross_signing_key(key) - except ValueError as e: - logger.warning( - "Invalid %s key retrieved: %s - %s %s", key_type, key, type(e), e, - ) - raise SynapseError( - 502, "Invalid %s key retrieved from database" % (key_type,) - ) - - # If we couldn't find the key locally, and we're looking for keys of - # another user then attempt to fetch the missing key from the remote - # user's server. - # - # We may run into this in possible edge cases where a user tries to - # cross-sign a remote user, but does not share any rooms with them yet. - # Thus, we would not have their key list yet. We fetch the key here, - # store it and notify clients of new, associated device IDs. - if ( - key is None - and not self.is_mine(user) - # We only get "master" and "self_signing" keys from remote servers - and key_type in ["master", "self_signing"] - ): - ( - key, - key_id, - verify_key, - ) = yield self._retrieve_cross_signing_keys_for_remote_user(user, key_type) - if key is None: - logger.debug("No %s key found for %s", key_type, user_id) + logger.debug("no %s key found for %s", key_type, user_id) raise NotFoundError("No %s key found for %s" % (key_type, user_id)) - + key_id, verify_key = get_verify_key_from_cross_signing_key(key) return key, key_id, verify_key - @defer.inlineCallbacks - def _retrieve_cross_signing_keys_for_remote_user( - self, user: UserID, desired_key_type: str, - ): - """Queries cross-signing keys for a remote user and saves them to the database - - Only the key specified by `key_type` will be returned, while all retrieved keys - will be saved regardless - - Args: - user: The user to query remote keys for - desired_key_type: The type of key to receive. One of "master", "self_signing" - - Returns: - Deferred[Tuple[Optional[Dict], Optional[str], Optional[VerifyKey]]]: A tuple - of the retrieved key content, the key's ID and the matching VerifyKey. - If the key cannot be retrieved, all values in the tuple will instead be None. - """ - try: - remote_result = yield self.federation.query_user_devices( - user.domain, user.to_string() - ) - except Exception as e: - logger.warning( - "Unable to query %s for cross-signing keys of user %s: %s %s", - user.domain, - user.to_string(), - type(e), - e, - ) - return None, None, None - - # Process each of the retrieved cross-signing keys - final_key = None - final_key_id = None - final_verify_key = None - device_ids = [] - for key_type in ["master", "self_signing"]: - key_content = remote_result.get(key_type + "_key") - if not key_content: - continue - - # At the same time, store this key in the db for - # subsequent queries - yield self.store.set_e2e_cross_signing_key( - user.to_string(), key_type, key_content - ) - - # Note down the device ID attached to this key - try: - # verify_key is a VerifyKey from signedjson, which uses - # .version to denote the portion of the key ID after the - # algorithm and colon, which is the device ID - key_id, verify_key = get_verify_key_from_cross_signing_key(key_content) - except ValueError as e: - logger.warning( - "Invalid %s key retrieved from remote %s: %s - %s %s", - user.domain, - key_type, - key_content, - type(e), - e, - ) - continue - device_ids.append(verify_key.version) - - # If this is the desired key type, save it and its ID/VerifyKey - if key_type == desired_key_type: - final_key = key_content - final_verify_key = verify_key - final_key_id = key_id - - # Notify clients that new devices for this user have been discovered - if device_ids: - yield self.device_handler.notify_device_update(user.to_string(), device_ids) - - return final_key, final_key_id, final_verify_key - def _check_cross_signing_key(key, user_id, key_type, signing_key=None): """Check a cross-signing key uploaded by a user. Performs some basic sanity From 5debf3071cc04896b2ab6169dcfd9ea2586200c7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 4 May 2020 16:44:21 +0100 Subject: [PATCH 105/178] Fix redis password support --- synapse/replication/tcp/redis.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index 617e860f95..41c623d737 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -61,6 +61,7 @@ class RedisSubscriber(txredisapi.SubscriberProtocol, AbstractConnection): outbound_redis_connection = None # type: txredisapi.RedisProtocol def connectionMade(self): + super().connectionMade() logger.info("Connected to redis instance") self.subscribe(self.stream_name) self.send_command(ReplicateCommand()) @@ -119,6 +120,7 @@ async def handle_command(self, cmd: Command): logger.warning("Unhandled command: %r", cmd) def connectionLost(self, reason): + super().connectionLost(reason) logger.info("Lost connection to redis instance") self.handler.lost_connection(self) @@ -189,5 +191,6 @@ def buildProtocol(self, addr): p.handler = self.handler p.outbound_redis_connection = self.outbound_redis_connection p.stream_name = self.stream_name + p.password = self.password return p From 323cfe3efb433d4d34d3fe428935692b55984145 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 6 May 2020 12:14:01 +0100 Subject: [PATCH 106/178] fix bad merge --- synapse/handlers/room_member.py | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index cdae8e57bc..79d69f9b3b 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -268,28 +268,27 @@ async def update_membership( ): key = (room_id,) -<<<<<<< HEAD as_id = object() if requester.app_service: as_id = requester.app_service.id then = self.clock.time_msec() - with (yield self.member_limiter.queue(as_id)): + with (await self.member_limiter.queue(as_id)): diff = self.clock.time_msec() - then if diff > 80 * 1000: # haproxy would have timed the request out anyway... raise SynapseError(504, "took to long to process") - with (yield self.member_linearizer.queue(key)): + with (await self.member_linearizer.queue(key)): diff = self.clock.time_msec() - then if diff > 80 * 1000: # haproxy would have timed the request out anyway... raise SynapseError(504, "took to long to process") - result = yield self._update_membership( + result = await self._update_membership( requester, target, room_id, @@ -301,21 +300,6 @@ async def update_membership( content=content, require_consent=require_consent, ) -======= - with (await self.member_linearizer.queue(key)): - result = await self._update_membership( - requester, - target, - room_id, - action, - txn_id=txn_id, - remote_room_hosts=remote_room_hosts, - third_party_signed=third_party_signed, - ratelimit=ratelimit, - content=content, - require_consent=require_consent, - ) ->>>>>>> release-v1.13.0 return result From 4d3ebc36203ffaf2079024335d4bb285f1c6a00e Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 7 May 2020 08:34:12 -0400 Subject: [PATCH 107/178] Disable validation that a UI authentication session has not been modified during a request cycle. Partial backout of 1c1242acba9694a3a4b1eb3b14ec0bac11ee4ff8 (#7068) --- synapse/handlers/auth.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 7613e5b6ab..a167498add 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -329,18 +329,6 @@ async def check_auth( # isn't arbitrary. clientdict = session.clientdict - # Ensure that the queried operation does not vary between stages of - # the UI authentication session. This is done by generating a stable - # comparator based on the URI, method, and body (minus the auth dict) - # and storing it during the initial query. Subsequent queries ensure - # that this comparator has not changed. - comparator = (uri, method, clientdict) - if (session.uri, session.method, session.clientdict) != comparator: - raise SynapseError( - 403, - "Requested operation has changed during the UI authentication session.", - ) - if not authdict: raise InteractiveAuthIncompleteError( self._auth_dict_for_flows(flows, session.session_id) From c4c84b67d58487247c8591caa4af2f459db78d7c Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 7 May 2020 10:03:57 -0400 Subject: [PATCH 108/178] Disable a failing test. --- tests/rest/client/v2_alpha/test_auth.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/rest/client/v2_alpha/test_auth.py b/tests/rest/client/v2_alpha/test_auth.py index 587be7b2e7..efc20f86aa 100644 --- a/tests/rest/client/v2_alpha/test_auth.py +++ b/tests/rest/client/v2_alpha/test_auth.py @@ -182,6 +182,9 @@ def test_cannot_change_operation(self): self.render(request) self.assertEqual(channel.code, 403) + # This behavior is currently disabled. + test_cannot_change_operation.skip = True + def test_complete_operation_unknown_session(self): """ Attempting to mark an invalid session as complete should error. From d7c7f64f176ab73cfdcc1a26d71cfbffc49c506c Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 7 May 2020 10:07:09 -0400 Subject: [PATCH 109/178] Propagate changes to the client dict to the database. --- synapse/handlers/auth.py | 4 +++- synapse/storage/data_stores/main/ui_auth.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index a167498add..1d779d2978 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -317,7 +317,7 @@ async def check_auth( except StoreError: raise SynapseError(400, "Unknown session ID: %s" % (sid,)) - if not clientdict: + if clientdict: # This was designed to allow the client to omit the parameters # and just supply the session in subsequent calls so it split # auth between devices by just sharing the session, (eg. so you @@ -327,6 +327,8 @@ async def check_auth( # on a homeserver. # Revisit: Assuming the REST APIs do sensible validation, the data # isn't arbitrary. + await self.store.set_ui_auth_clientdict(sid, clientdict) + else: clientdict = session.clientdict if not authdict: diff --git a/synapse/storage/data_stores/main/ui_auth.py b/synapse/storage/data_stores/main/ui_auth.py index c8eebc9378..1d8ee22fb1 100644 --- a/synapse/storage/data_stores/main/ui_auth.py +++ b/synapse/storage/data_stores/main/ui_auth.py @@ -172,6 +172,27 @@ async def get_completed_ui_auth_stages( return results + async def set_ui_auth_clientdict( + self, session_id: str, clientdict: JsonDict + ) -> None: + """ + Store an updated clientdict for a given session ID. + + Args: + session_id: The ID of this session as returned from check_auth + clientdict: + The dictionary from the client root level, not the 'auth' key. + """ + # The clientdict gets stored as JSON. + clientdict_json = json.dumps(clientdict) + + self.db.simple_update_one( + table="ui_auth_sessions", + keyvalues={"session_id": session_id}, + updatevalues={"clientdict": clientdict_json}, + desc="set_ui_auth_client_dict", + ) + async def set_ui_auth_session_data(self, session_id: str, key: str, value: Any): """ Store a key-value pair into the sessions data associated with this From 6610343332bd16cfebe94fb93fe7b55c7362e28a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 May 2020 13:08:14 +0100 Subject: [PATCH 110/178] Revert emergency registration patches Revert "Merge remote-tracking branch 'origin/clokep/no-validate-ui-auth-sess' into matrix-org-hotfixes" This reverts commit 5adad58d959ed0b249d43a9df81f034edc8876e7, reversing changes made to 617541c4c6f9dea1ac1ed0a8f1ab848507457e23. --- synapse/handlers/auth.py | 4 +--- synapse/storage/data_stores/main/ui_auth.py | 21 --------------------- tests/rest/client/v2_alpha/test_auth.py | 3 --- 3 files changed, 1 insertion(+), 27 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 1d779d2978..a167498add 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -317,7 +317,7 @@ async def check_auth( except StoreError: raise SynapseError(400, "Unknown session ID: %s" % (sid,)) - if clientdict: + if not clientdict: # This was designed to allow the client to omit the parameters # and just supply the session in subsequent calls so it split # auth between devices by just sharing the session, (eg. so you @@ -327,8 +327,6 @@ async def check_auth( # on a homeserver. # Revisit: Assuming the REST APIs do sensible validation, the data # isn't arbitrary. - await self.store.set_ui_auth_clientdict(sid, clientdict) - else: clientdict = session.clientdict if not authdict: diff --git a/synapse/storage/data_stores/main/ui_auth.py b/synapse/storage/data_stores/main/ui_auth.py index 1d8ee22fb1..c8eebc9378 100644 --- a/synapse/storage/data_stores/main/ui_auth.py +++ b/synapse/storage/data_stores/main/ui_auth.py @@ -172,27 +172,6 @@ async def get_completed_ui_auth_stages( return results - async def set_ui_auth_clientdict( - self, session_id: str, clientdict: JsonDict - ) -> None: - """ - Store an updated clientdict for a given session ID. - - Args: - session_id: The ID of this session as returned from check_auth - clientdict: - The dictionary from the client root level, not the 'auth' key. - """ - # The clientdict gets stored as JSON. - clientdict_json = json.dumps(clientdict) - - self.db.simple_update_one( - table="ui_auth_sessions", - keyvalues={"session_id": session_id}, - updatevalues={"clientdict": clientdict_json}, - desc="set_ui_auth_client_dict", - ) - async def set_ui_auth_session_data(self, session_id: str, key: str, value: Any): """ Store a key-value pair into the sessions data associated with this diff --git a/tests/rest/client/v2_alpha/test_auth.py b/tests/rest/client/v2_alpha/test_auth.py index efc20f86aa..587be7b2e7 100644 --- a/tests/rest/client/v2_alpha/test_auth.py +++ b/tests/rest/client/v2_alpha/test_auth.py @@ -182,9 +182,6 @@ def test_cannot_change_operation(self): self.render(request) self.assertEqual(channel.code, 403) - # This behavior is currently disabled. - test_cannot_change_operation.skip = True - def test_complete_operation_unknown_session(self): """ Attempting to mark an invalid session as complete should error. From 7ff7a415d17128fa50dd05b8b033be3ee811770c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 May 2020 13:08:48 +0100 Subject: [PATCH 111/178] Revert emergency registration patches Revert "Merge commit '4d3ebc' into matrix-org-hotfixes" This reverts commit 617541c4c6f9dea1ac1ed0a8f1ab848507457e23, reversing changes made to ae4f6140f134b8a9296b35ff15b37641912c76ec. --- synapse/handlers/auth.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index a167498add..7613e5b6ab 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -329,6 +329,18 @@ async def check_auth( # isn't arbitrary. clientdict = session.clientdict + # Ensure that the queried operation does not vary between stages of + # the UI authentication session. This is done by generating a stable + # comparator based on the URI, method, and body (minus the auth dict) + # and storing it during the initial query. Subsequent queries ensure + # that this comparator has not changed. + comparator = (uri, method, clientdict) + if (session.uri, session.method, session.clientdict) != comparator: + raise SynapseError( + 403, + "Requested operation has changed during the UI authentication session.", + ) + if not authdict: raise InteractiveAuthIncompleteError( self._auth_dict_for_flows(flows, session.session_id) From 84639b32ae9f99de0405186d6568fd510caff21e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 20 May 2020 22:11:30 +0100 Subject: [PATCH 112/178] stub out GET presence requests --- synapse/app/generic_worker.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 667ad20428..9d9849365a 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -140,31 +140,18 @@ class PresenceStatusStubServlet(RestServlet): """If presence is disabled this servlet can be used to stub out setting - presence status, while proxying the getters to the master instance. + presence status. """ PATTERNS = client_patterns("/presence/(?P[^/]*)/status") def __init__(self, hs): super(PresenceStatusStubServlet, self).__init__() - self.http_client = hs.get_simple_http_client() self.auth = hs.get_auth() - self.main_uri = hs.config.worker_main_http_uri async def on_GET(self, request, user_id): - # Pass through the auth headers, if any, in case the access token - # is there. - auth_headers = request.requestHeaders.getRawHeaders("Authorization", []) - headers = {"Authorization": auth_headers} - - try: - result = await self.http_client.get_json( - self.main_uri + request.uri.decode("ascii"), headers=headers - ) - except HttpResponseException as e: - raise e.to_synapse_error() - - return 200, result + await self.auth.get_user_by_req(request) + return 200, {"state": "offline"} async def on_PUT(self, request, user_id): await self.auth.get_user_by_req(request) From b3a9ad124c703b51238a33ed33aaa0f95041df7c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 20 May 2020 22:17:39 +0100 Subject: [PATCH 113/178] Fix field name in stubbed out presence servlet --- synapse/app/generic_worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 9d9849365a..d78c79e0f9 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -27,7 +27,7 @@ import synapse import synapse.events from synapse.api.constants import EventTypes -from synapse.api.errors import HttpResponseException, SynapseError +from synapse.api.errors import SynapseError from synapse.api.urls import ( CLIENT_API_PREFIX, FEDERATION_PREFIX, @@ -151,7 +151,7 @@ def __init__(self, hs): async def on_GET(self, request, user_id): await self.auth.get_user_by_req(request) - return 200, {"state": "offline"} + return 200, {"presence": "offline"} async def on_PUT(self, request, user_id): await self.auth.get_user_by_req(request) From 1c347c84bf2287b364932c62e75b853e7af996e3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 20 May 2020 23:33:13 +0100 Subject: [PATCH 114/178] inline some config references --- synapse/handlers/message.py | 8 +++++--- .../resource_limits_server_notices.py | 15 +++++++-------- .../data_stores/main/monthly_active_users.py | 19 +++++++++++-------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index a622a600b4..66b46bd59f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -403,8 +403,10 @@ def __init__(self, hs): if self._block_events_without_consent_error: self._consent_uri_builder = ConsentURIBuilder(self.config) + self._is_worker_app = self.config.worker_app is not None + if ( - not self.config.worker_app + not self._is_worker_app and self.config.cleanup_extremities_with_dummy_events ): self.clock.looping_call( @@ -824,7 +826,7 @@ async def handle_new_client_event( success = False try: # If we're a worker we need to hit out to the master. - if self.config.worker_app: + if self._is_worker_app: await self.send_event_to_master( event_id=event.event_id, store=self.store, @@ -890,7 +892,7 @@ async def persist_and_notify_client_event( This should only be run on master. """ - assert not self.config.worker_app + assert not self._is_worker_app if ratelimit: # We check if this is a room admin redacting an event so that we diff --git a/synapse/server_notices/resource_limits_server_notices.py b/synapse/server_notices/resource_limits_server_notices.py index d97166351e..73f2cedb5c 100644 --- a/synapse/server_notices/resource_limits_server_notices.py +++ b/synapse/server_notices/resource_limits_server_notices.py @@ -48,6 +48,12 @@ def __init__(self, hs): self._notifier = hs.get_notifier() + self._enabled = ( + hs.config.limit_usage_by_mau + and self._server_notices_manager.is_enabled() + and not hs.config.hs_disabled + ) + async def maybe_send_server_notice_to_user(self, user_id): """Check if we need to send a notice to this user, this will be true in two cases. @@ -61,14 +67,7 @@ async def maybe_send_server_notice_to_user(self, user_id): Returns: Deferred """ - if self._config.hs_disabled is True: - return - - if self._config.limit_usage_by_mau is False: - return - - if not self._server_notices_manager.is_enabled(): - # Don't try and send server notices unless they've been enabled + if not self._enabled: return timestamp = await self._store.user_last_seen_monthly_active(user_id) diff --git a/synapse/storage/data_stores/main/monthly_active_users.py b/synapse/storage/data_stores/main/monthly_active_users.py index 925bc5691b..a624d1f1b6 100644 --- a/synapse/storage/data_stores/main/monthly_active_users.py +++ b/synapse/storage/data_stores/main/monthly_active_users.py @@ -122,6 +122,10 @@ class MonthlyActiveUsersStore(MonthlyActiveUsersWorkerStore): def __init__(self, database: Database, db_conn, hs): super(MonthlyActiveUsersStore, self).__init__(database, db_conn, hs) + self._limit_usage_by_mau = hs.config.limit_usage_by_mau + self._mau_stats_only = hs.config.mau_stats_only + self._max_mau_value = hs.config.max_mau_value + # Do not add more reserved users than the total allowable number # cur = LoggingTransaction( self.db.new_transaction( @@ -130,7 +134,7 @@ def __init__(self, database: Database, db_conn, hs): [], [], self._initialise_reserved_users, - hs.config.mau_limits_reserved_threepids[: self.hs.config.max_mau_value], + hs.config.mau_limits_reserved_threepids[: self._max_mau_value], ) def _initialise_reserved_users(self, txn, threepids): @@ -191,8 +195,7 @@ def _reap_users(txn, reserved_users): txn.execute(sql, query_args) - max_mau_value = self.hs.config.max_mau_value - if self.hs.config.limit_usage_by_mau: + if self._limit_usage_by_mau: # If MAU user count still exceeds the MAU threshold, then delete on # a least recently active basis. # Note it is not possible to write this query using OFFSET due to @@ -210,13 +213,13 @@ def _reap_users(txn, reserved_users): LIMIT ? ) """ - txn.execute(sql, (max_mau_value,)) + txn.execute(sql, ((self._max_mau_value),)) # Need if/else since 'AND user_id NOT IN ({})' fails on Postgres # when len(reserved_users) == 0. Works fine on sqlite. else: # Must be >= 0 for postgres num_of_non_reserved_users_to_remove = max( - max_mau_value - len(reserved_users), 0 + self._max_mau_value - len(reserved_users), 0 ) # It is important to filter reserved users twice to guard @@ -335,7 +338,7 @@ def populate_monthly_active_users(self, user_id): Args: user_id(str): the user_id to query """ - if self.hs.config.limit_usage_by_mau or self.hs.config.mau_stats_only: + if self._limit_usage_by_mau or self._mau_stats_only: # Trial users and guests should not be included as part of MAU group is_guest = yield self.is_guest(user_id) if is_guest: @@ -356,11 +359,11 @@ def populate_monthly_active_users(self, user_id): # In the case where mau_stats_only is True and limit_usage_by_mau is # False, there is no point in checking get_monthly_active_count - it # adds no value and will break the logic if max_mau_value is exceeded. - if not self.hs.config.limit_usage_by_mau: + if not self._limit_usage_by_mau: yield self.upsert_monthly_active_user(user_id) else: count = yield self.get_monthly_active_count() - if count < self.hs.config.max_mau_value: + if count < self._max_mau_value: yield self.upsert_monthly_active_user(user_id) elif now - last_seen_timestamp > LAST_SEEN_GRANULARITY: yield self.upsert_monthly_active_user(user_id) From 104c4902742461bfcbb8be645f4af782680b8a6e Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Fri, 18 Sep 2020 14:51:11 +0100 Subject: [PATCH 115/178] Use _check_sigs_and_hash_and_fetch to validate backfill requests (#8350) This is a bit of a hack, as `_check_sigs_and_hash_and_fetch` is intended for attempting to pull an event from the database/(re)pull it from the server that originally sent the event if checking the signature of the event fails. During backfill we *know* that we won't have the event in our database, however it is still useful to be able to query the original sending server as the server we're backfilling from may be acting maliciously. The main benefit and reason for this change however is that `_check_sigs_and_hash_and_fetch` will drop an event during backfill if it cannot be successfully validated, whereas the current code will simply fail the backfill request - resulting in the client's /messages request silently being dropped. This is a quick patch to fix backfilling rooms that contain malformed events. A better implementation in planned in future. --- changelog.d/8350.bugfix | 1 + synapse/federation/federation_client.py | 8 +++----- 2 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 changelog.d/8350.bugfix diff --git a/changelog.d/8350.bugfix b/changelog.d/8350.bugfix new file mode 100644 index 0000000000..0e493c0282 --- /dev/null +++ b/changelog.d/8350.bugfix @@ -0,0 +1 @@ +Partially mitigate bug where newly joined servers couldn't get past events in a room when there is a malformed event. \ No newline at end of file diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index a2e8d96ea2..d42930d1b9 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -217,11 +217,9 @@ async def backfill( for p in transaction_data["pdus"] ] - # FIXME: We should handle signature failures more gracefully. - pdus[:] = await make_deferred_yieldable( - defer.gatherResults( - self._check_sigs_and_hashes(room_version, pdus), consumeErrors=True, - ).addErrback(unwrapFirstError) + # Check signatures and hash of pdus, removing any from the list that fail checks + pdus[:] = await self._check_sigs_and_hash_and_fetch( + dest, pdus, outlier=True, room_version=room_version ) return pdus From 9eea5c43afb0e7e3fde47576be0ce931f17559f7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 18 Sep 2020 14:25:52 +0100 Subject: [PATCH 116/178] Intelligently select extremities used in backfill. (#8349) Instead of just using the most recent extremities let's pick the ones that will give us results that the pagination request cares about, i.e. pick extremities only if they have a smaller depth than the pagination token. This is useful when we fail to backfill an extremity, as we no longer get stuck requesting that same extremity repeatedly. --- changelog.d/8349.bugfix | 1 + synapse/handlers/federation.py | 65 +++++++++++++++++++++--- synapse/handlers/pagination.py | 8 +-- synapse/storage/databases/main/stream.py | 13 ++--- 4 files changed, 67 insertions(+), 20 deletions(-) create mode 100644 changelog.d/8349.bugfix diff --git a/changelog.d/8349.bugfix b/changelog.d/8349.bugfix new file mode 100644 index 0000000000..cf2f531b14 --- /dev/null +++ b/changelog.d/8349.bugfix @@ -0,0 +1 @@ +Fix a longstanding bug where back pagination over federation could get stuck if it failed to handle a received event. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 43f2986f89..014dab2940 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -943,15 +943,26 @@ async def backfill(self, dest, room_id, limit, extremities): return events - async def maybe_backfill(self, room_id, current_depth): + async def maybe_backfill( + self, room_id: str, current_depth: int, limit: int + ) -> bool: """Checks the database to see if we should backfill before paginating, and if so do. + + Args: + room_id + current_depth: The depth from which we're paginating from. This is + used to decide if we should backfill and what extremities to + use. + limit: The number of events that the pagination request will + return. This is used as part of the heuristic to decide if we + should back paginate. """ extremities = await self.store.get_oldest_events_with_depth_in_room(room_id) if not extremities: logger.debug("Not backfilling as no extremeties found.") - return + return False # We only want to paginate if we can actually see the events we'll get, # as otherwise we'll just spend a lot of resources to get redacted @@ -1004,16 +1015,54 @@ async def maybe_backfill(self, room_id, current_depth): sorted_extremeties_tuple = sorted(extremities.items(), key=lambda e: -int(e[1])) max_depth = sorted_extremeties_tuple[0][1] + # If we're approaching an extremity we trigger a backfill, otherwise we + # no-op. + # + # We chose twice the limit here as then clients paginating backwards + # will send pagination requests that trigger backfill at least twice + # using the most recent extremity before it gets removed (see below). We + # chose more than one times the limit in case of failure, but choosing a + # much larger factor will result in triggering a backfill request much + # earlier than necessary. + if current_depth - 2 * limit > max_depth: + logger.debug( + "Not backfilling as we don't need to. %d < %d - 2 * %d", + max_depth, + current_depth, + limit, + ) + return False + + logger.debug( + "room_id: %s, backfill: current_depth: %s, max_depth: %s, extrems: %s", + room_id, + current_depth, + max_depth, + sorted_extremeties_tuple, + ) + + # We ignore extremities that have a greater depth than our current depth + # as: + # 1. we don't really care about getting events that have happened + # before our current position; and + # 2. we have likely previously tried and failed to backfill from that + # extremity, so to avoid getting "stuck" requesting the same + # backfill repeatedly we drop those extremities. + filtered_sorted_extremeties_tuple = [ + t for t in sorted_extremeties_tuple if int(t[1]) <= current_depth + ] + + # However, we need to check that the filtered extremities are non-empty. + # If they are empty then either we can a) bail or b) still attempt to + # backill. We opt to try backfilling anyway just in case we do get + # relevant events. + if filtered_sorted_extremeties_tuple: + sorted_extremeties_tuple = filtered_sorted_extremeties_tuple + # We don't want to specify too many extremities as it causes the backfill # request URI to be too long. extremities = dict(sorted_extremeties_tuple[:5]) - if current_depth > max_depth: - logger.debug( - "Not backfilling as we don't need to. %d < %d", max_depth, current_depth - ) - return - # Now we need to decide which hosts to hit first. # First we try hosts that are already in the room diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 34ed0e2921..6067585f9b 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -362,9 +362,9 @@ async def get_messages( # if we're going backwards, we might need to backfill. This # requires that we have a topo token. if room_token.topological: - max_topo = room_token.topological + curr_topo = room_token.topological else: - max_topo = await self.store.get_max_topological_token( + curr_topo = await self.store.get_current_topological_token( room_id, room_token.stream ) @@ -380,11 +380,11 @@ async def get_messages( leave_token = await self.store.get_topological_token_for_event( member_event_id ) - if RoomStreamToken.parse(leave_token).topological < max_topo: + if RoomStreamToken.parse(leave_token).topological < curr_topo: source_config.from_key = str(leave_token) await self.hs.get_handlers().federation_handler.maybe_backfill( - room_id, max_topo + room_id, curr_topo, limit=source_config.limit, ) events, next_key = await self.store.paginate_room_events( diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index be6df8a6d1..db20a3db30 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -648,23 +648,20 @@ async def get_topological_token_for_event(self, event_id: str) -> str: ) return "t%d-%d" % (row["topological_ordering"], row["stream_ordering"]) - async def get_max_topological_token(self, room_id: str, stream_key: int) -> int: - """Get the max topological token in a room before the given stream + async def get_current_topological_token(self, room_id: str, stream_key: int) -> int: + """Gets the topological token in a room after or at the given stream ordering. Args: room_id stream_key - - Returns: - The maximum topological token. """ sql = ( - "SELECT coalesce(max(topological_ordering), 0) FROM events" - " WHERE room_id = ? AND stream_ordering < ?" + "SELECT coalesce(MIN(topological_ordering), 0) FROM events" + " WHERE room_id = ? AND stream_ordering >= ?" ) row = await self.db_pool.execute( - "get_max_topological_token", None, sql, room_id, stream_key + "get_current_topological_token", None, sql, room_id, stream_key ) return row[0][0] if row else 0 From cfb3096e33d8087a4b5214945d0b24295bb1c56a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 1 Oct 2020 11:19:09 +0100 Subject: [PATCH 117/178] Revert federation-transaction-transmission backoff hacks This reverts b852a8247, 15b2a5081, 28889d8da. I don't think these patches are required any more, and if they are, they should be on mainline, not hidden in our hotfixes branch. Let's try backing them out: if that turns out to be an error, we can PR them properly. --- synapse/federation/sender/__init__.py | 20 +------------------ .../sender/per_destination_queue.py | 15 -------------- .../federation/sender/transaction_manager.py | 4 ---- 3 files changed, 1 insertion(+), 38 deletions(-) diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index b22869501c..552519e82c 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -151,25 +151,10 @@ def notify_new_events(self, current_id: int) -> None: "process_event_queue_for_federation", self._process_event_queue_loop ) - async def _process_event_queue_loop(self): - loop_start_time = self.clock.time_msec() + async def _process_event_queue_loop(self) -> None: try: self._is_processing = True while True: - # if we've been going around this loop for a long time without - # catching up, deprioritise transaction transmission. This should mean - # that events get batched into fewer transactions, which is more - # efficient, and hence give us a chance to catch up - if ( - self.clock.time_msec() - loop_start_time > 60 * 1000 - and not self._transaction_manager.deprioritise_transmission - ): - logger.warning( - "Event queue is getting behind: deprioritising transaction " - "transmission" - ) - self._transaction_manager.deprioritise_transmission = True - last_token = await self.store.get_federation_out_pos("events") next_token, events = await self.store.get_all_new_events_stream( last_token, self._last_poked_id, limit=100 @@ -279,9 +264,6 @@ async def handle_room_events(events: Iterable[EventBase]) -> None: finally: self._is_processing = False - if self._transaction_manager.deprioritise_transmission: - logger.info("Event queue caught up: re-prioritising transmission") - self._transaction_manager.deprioritise_transmission = False def _send_pdu(self, pdu: EventBase, destinations: Iterable[str]) -> None: # We loop through all destinations to see whether we already have diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index b4da52e7e6..defc228c23 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -15,7 +15,6 @@ # limitations under the License. import datetime import logging -import random from typing import TYPE_CHECKING, Dict, Hashable, Iterable, List, Tuple from prometheus_client import Counter @@ -40,8 +39,6 @@ # This is defined in the Matrix spec and enforced by the receiver. MAX_EDUS_PER_TRANSACTION = 100 -DEPRIORITISE_SLEEP_TIME = 10 - logger = logging.getLogger(__name__) @@ -223,18 +220,6 @@ async def _transaction_transmission_loop(self) -> None: pending_pdus = [] while True: - if self._transaction_manager.deprioritise_transmission: - # if the event-processing loop has got behind, sleep to give it - # a chance to catch up. Add some randomness so that the transmitters - # don't all wake up in sync. - sleeptime = random.uniform( - DEPRIORITISE_SLEEP_TIME, DEPRIORITISE_SLEEP_TIME * 2 - ) - logger.info( - "TX [%s]: sleeping for %f seconds", self._destination, sleeptime - ) - await self._clock.sleep(sleeptime) - # We have to keep 2 free slots for presence and rr_edus limit = MAX_EDUS_PER_TRANSACTION - 2 diff --git a/synapse/federation/sender/transaction_manager.py b/synapse/federation/sender/transaction_manager.py index cf472c9f15..c84072ab73 100644 --- a/synapse/federation/sender/transaction_manager.py +++ b/synapse/federation/sender/transaction_manager.py @@ -51,10 +51,6 @@ def __init__(self, hs: "synapse.server.HomeServer"): # HACK to get unique tx id self._next_txn_id = int(self.clock.time_msec()) - # the federation sender sometimes sets this to delay transaction transmission, - # if the sender gets behind. - self.deprioritise_transmission = False - @measure_func("_send_new_transaction") async def send_new_transaction( self, destination: str, pdus: List[EventBase], edus: List[Edu], From ac6c5f198ea28d621063015b8f61295ec69a2edc Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 1 Oct 2020 11:31:07 +0100 Subject: [PATCH 118/178] Remove dangling changelog.d files These result from PRs which were cherry-picked from release branches. --- changelog.d/6126.feature | 1 - changelog.d/6418.bugfix | 1 - changelog.d/8349.bugfix | 1 - changelog.d/8350.bugfix | 1 - 4 files changed, 4 deletions(-) delete mode 100644 changelog.d/6126.feature delete mode 100644 changelog.d/6418.bugfix delete mode 100644 changelog.d/8349.bugfix delete mode 100644 changelog.d/8350.bugfix diff --git a/changelog.d/6126.feature b/changelog.d/6126.feature deleted file mode 100644 index 1207ba6206..0000000000 --- a/changelog.d/6126.feature +++ /dev/null @@ -1 +0,0 @@ -Group events into larger federation transactions at times of high traffic. diff --git a/changelog.d/6418.bugfix b/changelog.d/6418.bugfix deleted file mode 100644 index a1f488d3a2..0000000000 --- a/changelog.d/6418.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix phone home stats reporting. diff --git a/changelog.d/8349.bugfix b/changelog.d/8349.bugfix deleted file mode 100644 index cf2f531b14..0000000000 --- a/changelog.d/8349.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a longstanding bug where back pagination over federation could get stuck if it failed to handle a received event. diff --git a/changelog.d/8350.bugfix b/changelog.d/8350.bugfix deleted file mode 100644 index 0e493c0282..0000000000 --- a/changelog.d/8350.bugfix +++ /dev/null @@ -1 +0,0 @@ -Partially mitigate bug where newly joined servers couldn't get past events in a room when there is a malformed event. \ No newline at end of file From 607367aeb1c9bf6366ebdf093d0bdba37891fe1b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 1 Oct 2020 11:43:16 +0100 Subject: [PATCH 119/178] Fix typo in comment I think this came from a bad merge --- synapse/storage/databases/main/monthly_active_users.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/monthly_active_users.py b/synapse/storage/databases/main/monthly_active_users.py index 097a16cb2e..e93aad33cd 100644 --- a/synapse/storage/databases/main/monthly_active_users.py +++ b/synapse/storage/databases/main/monthly_active_users.py @@ -206,7 +206,7 @@ def _reap_users(txn, reserved_users): # a least recently active basis. # Note it is not possible to write this query using OFFSET due to # incompatibilities in how sqlite and postgres support the feature. - # Sqlite requires 'LIMIT -1 OFFSET ?', the LIMIT must be presents, + # Sqlite requires 'LIMIT -1 OFFSET ?', the LIMIT must be present, # while Postgres does not require 'LIMIT', but also does not support # negative LIMIT values. So there is no way to write it that both can # support From e675bbcc49b04f9799bacb6e7f266573d8a117fc Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 1 Oct 2020 11:51:09 +0100 Subject: [PATCH 120/178] Remove redundant `EventCreationHandler._is_worker_app` attribute This was added in 1c347c84bf/#7544 as a temporary optimisation. That was never merged to develop, since it conflicted with #7492. The merge cf92310da forgot to remove it. --- synapse/handlers/message.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 16c367b30b..3a9183e0b0 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -412,10 +412,8 @@ def __init__(self, hs: "HomeServer"): if self._block_events_without_consent_error: self._consent_uri_builder = ConsentURIBuilder(self.config) - self._is_worker_app = self.config.worker_app is not None - if ( - not self._is_worker_app + not self.config.worker_app and self.config.cleanup_extremities_with_dummy_events ): self.clock.looping_call( From 1c22954668b67b21b4741026193c3eee2f61b161 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 1 Oct 2020 12:10:55 +0100 Subject: [PATCH 121/178] Revert "Temporary fix to ensure kde can contact matrix.org if stuff breaks" This reverts commit d90b0946ed775ca228895dd9f7e63bd16bed6391. We believe this is no longer required. --- synapse/http/federation/well_known_resolver.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/synapse/http/federation/well_known_resolver.py b/synapse/http/federation/well_known_resolver.py index fabe5d9939..a306faa267 100644 --- a/synapse/http/federation/well_known_resolver.py +++ b/synapse/http/federation/well_known_resolver.py @@ -110,10 +110,6 @@ async def get_well_known(self, server_name: bytes) -> WellKnownLookupResult: Returns: The result of the lookup """ - - if server_name == b"kde.org": - return WellKnownLookupResult(delegated_server=b"kde.modular.im:443") - try: prev_result, expiry, ttl = self._well_known_cache.get_with_expiry( server_name From b0a463f758a71810227c8aa5d92023a549e6810a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 1 Oct 2020 15:53:02 +0100 Subject: [PATCH 122/178] fix remote thumbnails? --- synapse/rest/media/v1/media_storage.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/synapse/rest/media/v1/media_storage.py b/synapse/rest/media/v1/media_storage.py index 5681677fc9..4827cb2a3f 100644 --- a/synapse/rest/media/v1/media_storage.py +++ b/synapse/rest/media/v1/media_storage.py @@ -143,12 +143,9 @@ async def fetch_media(self, file_info: FileInfo) -> Optional[Responder]: """ path = self._file_info_to_path(file_info) - local_path = os.path.join(self.local_media_directory, path) - if os.path.exists(local_path): - return FileResponder(open(local_path, "rb")) - # Fallback for paths without method names - # Should be removed in the future + # fallback for remote thumbnails with no method in the filename + legacy_path = None if file_info.thumbnail and file_info.server_name: legacy_path = self.filepaths.remote_media_thumbnail_rel_legacy( server_name=file_info.server_name, @@ -157,8 +154,19 @@ async def fetch_media(self, file_info: FileInfo) -> Optional[Responder]: height=file_info.thumbnail_height, content_type=file_info.thumbnail_type, ) + + local_path = os.path.join(self.local_media_directory, path) + if os.path.exists(local_path): + logger.debug("responding with local file %s", local_path) + return FileResponder(open(local_path, "rb")) + + if legacy_path: + logger.debug( + "local file %s did not exist; checking legacy name", local_path + ) legacy_local_path = os.path.join(self.local_media_directory, legacy_path) if os.path.exists(legacy_local_path): + logger.debug("responding with local file %s", legacy_local_path) return FileResponder(open(legacy_local_path, "rb")) for provider in self.storage_providers: @@ -166,6 +174,14 @@ async def fetch_media(self, file_info: FileInfo) -> Optional[Responder]: if res: logger.debug("Streaming %s from %s", path, provider) return res + if legacy_path: + logger.debug( + "Provider %s did not find %s; checking legacy name", provider, path + ) + res = await provider.fetch(legacy_path, file_info) + if res: + logger.debug("Streaming %s from %s", legacy_path, provider) + return res return None From 5ccc0785c1a4c18517582034b00526873d41e280 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 2 Oct 2020 12:30:49 +0100 Subject: [PATCH 123/178] Revert "fix remote thumbnails?" This has now been fixed by a different commit (73d93039f). This reverts commit b0a463f758a71810227c8aa5d92023a549e6810a. --- synapse/rest/media/v1/media_storage.py | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/synapse/rest/media/v1/media_storage.py b/synapse/rest/media/v1/media_storage.py index 4827cb2a3f..5681677fc9 100644 --- a/synapse/rest/media/v1/media_storage.py +++ b/synapse/rest/media/v1/media_storage.py @@ -143,9 +143,12 @@ async def fetch_media(self, file_info: FileInfo) -> Optional[Responder]: """ path = self._file_info_to_path(file_info) + local_path = os.path.join(self.local_media_directory, path) + if os.path.exists(local_path): + return FileResponder(open(local_path, "rb")) - # fallback for remote thumbnails with no method in the filename - legacy_path = None + # Fallback for paths without method names + # Should be removed in the future if file_info.thumbnail and file_info.server_name: legacy_path = self.filepaths.remote_media_thumbnail_rel_legacy( server_name=file_info.server_name, @@ -154,19 +157,8 @@ async def fetch_media(self, file_info: FileInfo) -> Optional[Responder]: height=file_info.thumbnail_height, content_type=file_info.thumbnail_type, ) - - local_path = os.path.join(self.local_media_directory, path) - if os.path.exists(local_path): - logger.debug("responding with local file %s", local_path) - return FileResponder(open(local_path, "rb")) - - if legacy_path: - logger.debug( - "local file %s did not exist; checking legacy name", local_path - ) legacy_local_path = os.path.join(self.local_media_directory, legacy_path) if os.path.exists(legacy_local_path): - logger.debug("responding with local file %s", legacy_local_path) return FileResponder(open(legacy_local_path, "rb")) for provider in self.storage_providers: @@ -174,14 +166,6 @@ async def fetch_media(self, file_info: FileInfo) -> Optional[Responder]: if res: logger.debug("Streaming %s from %s", path, provider) return res - if legacy_path: - logger.debug( - "Provider %s did not find %s; checking legacy name", provider, path - ) - res = await provider.fetch(legacy_path, file_info) - if res: - logger.debug("Streaming %s from %s", legacy_path, provider) - return res return None From cd2f831b9da91829efab01f31921fbd7fb738300 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 12 Oct 2020 19:09:30 +0100 Subject: [PATCH 124/178] block membership events from spammy freenode bridge --- synapse/rest/client/v1/room.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index b63389e5fe..862d05e3ca 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -291,6 +291,8 @@ def register(self, http_server): async def on_POST(self, request, room_identifier, txn_id=None): requester = await self.auth.get_user_by_req(request, allow_guest=True) + if requester.app_service and requester.app_service.id == "irc-freenode": + raise SynapseError(400, "too much spam") try: content = parse_json_object_from_request(request) @@ -720,6 +722,8 @@ def register(self, http_server): async def on_POST(self, request, room_id, membership_action, txn_id=None): requester = await self.auth.get_user_by_req(request, allow_guest=True) + if requester.app_service and requester.app_service.id == "irc-freenode": + raise SynapseError(400, "too much spam") if requester.is_guest and membership_action not in { Membership.JOIN, From bdbe2b12c216287708d7dce11c65c58cdb1d706b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 13 Oct 2020 17:10:45 +0100 Subject: [PATCH 125/178] Revert "block membership events from spammy freenode bridge" This reverts commit cd2f831b9da91829efab01f31921fbd7fb738300. --- synapse/rest/client/v1/room.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index d2349ecfdd..00b4397082 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -290,8 +290,6 @@ def register(self, http_server): async def on_POST(self, request, room_identifier, txn_id=None): requester = await self.auth.get_user_by_req(request, allow_guest=True) - if requester.app_service and requester.app_service.id == "irc-freenode": - raise SynapseError(400, "too much spam") try: content = parse_json_object_from_request(request) @@ -721,8 +719,6 @@ def register(self, http_server): async def on_POST(self, request, room_id, membership_action, txn_id=None): requester = await self.auth.get_user_by_req(request, allow_guest=True) - if requester.app_service and requester.app_service.id == "irc-freenode": - raise SynapseError(400, "too much spam") if requester.is_guest and membership_action not in { Membership.JOIN, From d60af9305a07fadcf0270d1887c5b7d063834967 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 28 Oct 2020 11:58:47 +0000 Subject: [PATCH 126/178] Patch to temporarily drop cross-user m.key_share_requests (#8675) Cross-user `m.key_share_requests` are a relatively new `to_device` message that allows user to re-request session keys for a message from another user if they were otherwise unable to retrieve them. Unfortunately, these have had performance concerns on matrix.org. This is a temporary patch to disable them while we investigate a better solution. --- changelog.d/8675.misc | 1 + synapse/federation/federation_server.py | 4 ++++ synapse/handlers/devicemessage.py | 4 ++++ 3 files changed, 9 insertions(+) create mode 100644 changelog.d/8675.misc diff --git a/changelog.d/8675.misc b/changelog.d/8675.misc new file mode 100644 index 0000000000..7ffe38b7d9 --- /dev/null +++ b/changelog.d/8675.misc @@ -0,0 +1 @@ +Temporarily drop cross-user m.room_key_request to_device messages over performance concerns. diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 23278e36b7..b7459a1d87 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -915,6 +915,10 @@ async def on_edu(self, edu_type: str, origin: str, content: dict): if not self.config.use_presence and edu_type == "m.presence": return + # Temporary patch to drop cross-user key share requests + if edu_type == "m.room_key_request": + return + # Check if we have a handler on this instance handler = self.edu_handlers.get(edu_type) if handler: diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 9cac5a8463..5aa56013a4 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -153,6 +153,10 @@ async def send_device_message( local_messages = {} remote_messages = {} # type: Dict[str, Dict[str, Dict[str, JsonDict]]] for user_id, by_device in messages.items(): + # Temporary patch to disable sending local cross-user key requests. + if message_type == "m.room_key_request" and user_id != sender_user_id: + continue + # we use UserID.from_string to catch invalid user ids if self.is_mine(UserID.from_string(user_id)): messages_by_device = { From 8373e6254f683e4649a941f951d30b1255237264 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sat, 31 Oct 2020 10:50:28 +0000 Subject: [PATCH 127/178] Fix SIGHUP handler Fixes: ``` builtins.TypeError: _reload_logging_config() takes 1 positional argument but 2 were given ``` --- changelog.d/8697.misc | 1 + synapse/app/_base.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 changelog.d/8697.misc diff --git a/changelog.d/8697.misc b/changelog.d/8697.misc new file mode 100644 index 0000000000..7982a4e46d --- /dev/null +++ b/changelog.d/8697.misc @@ -0,0 +1 @@ + Re-organize the structured logging code to separate the TCP transport handling from the JSON formatting. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index f6f7b2bf42..9c8dc785c6 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -49,7 +49,6 @@ def register_sighup(func, *args, **kwargs): Args: func (function): Function to be called when sent a SIGHUP signal. - Will be called with a single default argument, the homeserver. *args, **kwargs: args and kwargs to be passed to the target function. """ _sighup_callbacks.append((func, args, kwargs)) @@ -251,13 +250,13 @@ def handle_sighup(*args, **kwargs): sdnotify(b"RELOADING=1") for i, args, kwargs in _sighup_callbacks: - i(hs, *args, **kwargs) + i(*args, **kwargs) sdnotify(b"READY=1") signal.signal(signal.SIGHUP, handle_sighup) - register_sighup(refresh_certificate) + register_sighup(refresh_certificate, hs) # Load the certificate from disk. refresh_certificate(hs) From f4f65f4e992cf27ca76be96af1831f9dc41f4759 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 29 Dec 2020 07:40:12 -0500 Subject: [PATCH 128/178] Allow redacting events on workers (#8994) Adds the redacts endpoint to workers that have the client listener. --- changelog.d/8994.feature | 1 + docs/workers.md | 1 + synapse/app/generic_worker.py | 31 ++++--------------------------- synapse/rest/client/v1/room.py | 17 ++++++++++------- 4 files changed, 16 insertions(+), 34 deletions(-) create mode 100644 changelog.d/8994.feature diff --git a/changelog.d/8994.feature b/changelog.d/8994.feature new file mode 100644 index 0000000000..76aeb185cb --- /dev/null +++ b/changelog.d/8994.feature @@ -0,0 +1 @@ +Allow running the redact endpoint on workers. diff --git a/docs/workers.md b/docs/workers.md index efe97af31a..298adf8695 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -229,6 +229,7 @@ expressions: ^/_matrix/client/(r0|unstable)/auth/.*/fallback/web$ # Event sending requests + ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/redact ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state/ ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$ diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index aa12c74358..fa23d9bb20 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -89,7 +89,7 @@ ToDeviceStream, ) from synapse.rest.admin import register_servlets_for_media_repo -from synapse.rest.client.v1 import events +from synapse.rest.client.v1 import events, room from synapse.rest.client.v1.initial_sync import InitialSyncRestServlet from synapse.rest.client.v1.login import LoginRestServlet from synapse.rest.client.v1.profile import ( @@ -98,20 +98,6 @@ ProfileRestServlet, ) from synapse.rest.client.v1.push_rule import PushRuleRestServlet -from synapse.rest.client.v1.room import ( - JoinedRoomMemberListRestServlet, - JoinRoomAliasServlet, - PublicRoomListRestServlet, - RoomEventContextServlet, - RoomInitialSyncRestServlet, - RoomMemberListRestServlet, - RoomMembershipRestServlet, - RoomMessageListRestServlet, - RoomSendEventRestServlet, - RoomStateEventRestServlet, - RoomStateRestServlet, - RoomTypingRestServlet, -) from synapse.rest.client.v1.voip import VoipRestServlet from synapse.rest.client.v2_alpha import groups, sync, user_directory from synapse.rest.client.v2_alpha._base import client_patterns @@ -512,12 +498,6 @@ def _listen_http(self, listener_config: ListenerConfig): elif name == "client": resource = JsonResource(self, canonical_json=False) - PublicRoomListRestServlet(self).register(resource) - RoomMemberListRestServlet(self).register(resource) - JoinedRoomMemberListRestServlet(self).register(resource) - RoomStateRestServlet(self).register(resource) - RoomEventContextServlet(self).register(resource) - RoomMessageListRestServlet(self).register(resource) RegisterRestServlet(self).register(resource) LoginRestServlet(self).register(resource) ThreepidRestServlet(self).register(resource) @@ -526,22 +506,19 @@ def _listen_http(self, listener_config: ListenerConfig): VoipRestServlet(self).register(resource) PushRuleRestServlet(self).register(resource) VersionsRestServlet(self).register(resource) - RoomSendEventRestServlet(self).register(resource) - RoomMembershipRestServlet(self).register(resource) - RoomStateEventRestServlet(self).register(resource) - JoinRoomAliasServlet(self).register(resource) + ProfileAvatarURLRestServlet(self).register(resource) ProfileDisplaynameRestServlet(self).register(resource) ProfileRestServlet(self).register(resource) KeyUploadServlet(self).register(resource) AccountDataServlet(self).register(resource) RoomAccountDataServlet(self).register(resource) - RoomTypingRestServlet(self).register(resource) sync.register_servlets(self, resource) events.register_servlets(self, resource) + room.register_servlets(self, resource, True) + room.register_deprecated_servlets(self, resource) InitialSyncRestServlet(self).register(resource) - RoomInitialSyncRestServlet(self).register(resource) user_directory.register_servlets(self, resource) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 93c06afe27..5647e8c577 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -963,25 +963,28 @@ def register_txn_path(servlet, regex_string, http_server, with_get=False): ) -def register_servlets(hs, http_server): +def register_servlets(hs, http_server, is_worker=False): RoomStateEventRestServlet(hs).register(http_server) - RoomCreateRestServlet(hs).register(http_server) RoomMemberListRestServlet(hs).register(http_server) JoinedRoomMemberListRestServlet(hs).register(http_server) RoomMessageListRestServlet(hs).register(http_server) JoinRoomAliasServlet(hs).register(http_server) - RoomForgetRestServlet(hs).register(http_server) RoomMembershipRestServlet(hs).register(http_server) RoomSendEventRestServlet(hs).register(http_server) PublicRoomListRestServlet(hs).register(http_server) RoomStateRestServlet(hs).register(http_server) RoomRedactEventRestServlet(hs).register(http_server) RoomTypingRestServlet(hs).register(http_server) - SearchRestServlet(hs).register(http_server) - JoinedRoomsRestServlet(hs).register(http_server) - RoomEventServlet(hs).register(http_server) RoomEventContextServlet(hs).register(http_server) - RoomAliasListServlet(hs).register(http_server) + + # Some servlets only get registered for the main process. + if not is_worker: + RoomCreateRestServlet(hs).register(http_server) + RoomForgetRestServlet(hs).register(http_server) + SearchRestServlet(hs).register(http_server) + JoinedRoomsRestServlet(hs).register(http_server) + RoomEventServlet(hs).register(http_server) + RoomAliasListServlet(hs).register(http_server) def register_deprecated_servlets(hs, http_server): From 5a4f09228d893fcc164ae47654446ff723ba1ccf Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 5 Jan 2021 13:52:36 +0000 Subject: [PATCH 129/178] Remove cache from room directory query results This reverts a285fe0. Hopefully the cache is no longer required, thanks to --- synapse/handlers/room_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index a2c0340a3c..70522e40fa 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -45,7 +45,7 @@ def __init__(self, hs: "HomeServer"): self.enable_room_list_search = hs.config.enable_room_list_search self.response_cache = ResponseCache( - hs, "room_list", timeout_ms=10 * 60 * 1000 + hs, "room_list" ) # type: ResponseCache[Tuple[Optional[int], Optional[str], ThirdPartyInstanceID]] self.remote_response_cache = ResponseCache( hs, "remote_room_list", timeout_ms=30 * 1000 From 3f6530ed5575a9ea574e026dbd4fe59caf3afebe Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sat, 6 Feb 2021 11:02:53 +0000 Subject: [PATCH 130/178] block groups requests to fosdem --- synapse/handlers/groups_local.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py index 71f11ef94a..df040a675e 100644 --- a/synapse/handlers/groups_local.py +++ b/synapse/handlers/groups_local.py @@ -42,6 +42,9 @@ async def f(self, group_id, *args, **kwargs): else: destination = get_domain_from_id(group_id) + if destination == "fosdem.org": + raise SynapseError(502, "Failed to contact group server") + try: return await getattr(self.transport_client, func_name)( destination, group_id, *args, **kwargs From 844b3e3f65eb5d6fb03f5ee94ef70befeffcdd6b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sat, 6 Feb 2021 12:03:46 +0000 Subject: [PATCH 131/178] Revert "block groups requests to fosdem" This reverts commit 3f6530ed5575a9ea574e026dbd4fe59caf3afebe. --- synapse/handlers/groups_local.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py index df040a675e..71f11ef94a 100644 --- a/synapse/handlers/groups_local.py +++ b/synapse/handlers/groups_local.py @@ -42,9 +42,6 @@ async def f(self, group_id, *args, **kwargs): else: destination = get_domain_from_id(group_id) - if destination == "fosdem.org": - raise SynapseError(502, "Failed to contact group server") - try: return await getattr(self.transport_client, func_name)( destination, group_id, *args, **kwargs From a4aa56a0eb46ab43683e2569d8b7ca52f9715afa Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 11 Feb 2021 16:06:29 +0000 Subject: [PATCH 132/178] Ensure that we never stop reconnecting to redis (#9391) --- changelog.d/9391.bugfix | 1 + synapse/replication/tcp/redis.py | 26 ++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 changelog.d/9391.bugfix diff --git a/changelog.d/9391.bugfix b/changelog.d/9391.bugfix new file mode 100644 index 0000000000..b5e68e2ac7 --- /dev/null +++ b/changelog.d/9391.bugfix @@ -0,0 +1 @@ +Fix bug where Synapse would occaisonally stop reconnecting after the connection was lost. diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index fdd087683b..89f8af0f36 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -15,8 +15,9 @@ import logging from inspect import isawaitable -from typing import TYPE_CHECKING, Optional, Type, cast +from typing import TYPE_CHECKING, Generic, Optional, Type, TypeVar, cast +import attr import txredisapi from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable @@ -42,6 +43,24 @@ logger = logging.getLogger(__name__) +T = TypeVar("T") +V = TypeVar("V") + + +@attr.s +class ConstantProperty(Generic[T, V]): + """A descriptor that returns the given constant, ignoring attempts to set + it. + """ + + constant = attr.ib() # type: V + + def __get__(self, obj: Optional[T], objtype: Type[T] = None) -> V: + return self.constant + + def __set__(self, obj: Optional[T], value: V): + pass + class RedisSubscriber(txredisapi.SubscriberProtocol, AbstractConnection): """Connection to redis subscribed to replication stream. @@ -195,6 +214,10 @@ class SynapseRedisFactory(txredisapi.RedisFactory): we detect dead connections. """ + # We want to *always* retry connecting, txredisapi will stop if there is a + # failure during certain operations, e.g. during AUTH. + continueTrying = cast(bool, ConstantProperty(True)) + def __init__( self, hs: "HomeServer", @@ -243,7 +266,6 @@ class RedisDirectTcpReplicationClientFactory(SynapseRedisFactory): """ maxDelay = 5 - continueTrying = True protocol = RedisSubscriber def __init__( From 5ee8a1c50a1b571a8a8704a59635232193b454f2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 18 Feb 2021 14:01:23 +0000 Subject: [PATCH 133/178] Redirect redirect requests if they arrive on the wrong URI --- synapse/rest/client/v1/login.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 6e2fbedd99..3e6a21e20f 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -354,6 +354,7 @@ def __init__(self, hs: "HomeServer"): hs.get_oidc_handler() self._sso_handler = hs.get_sso_handler() self._msc2858_enabled = hs.config.experimental.msc2858_enabled + self._public_baseurl = hs.config.public_baseurl def register(self, http_server: HttpServer) -> None: super().register(http_server) @@ -373,6 +374,28 @@ def register(self, http_server: HttpServer) -> None: async def on_GET( self, request: SynapseRequest, idp_id: Optional[str] = None ) -> None: + if not self._public_baseurl: + raise SynapseError(400, "SSO requires a valid public_baseurl") + + # if this isn't the expected hostname, redirect to the right one, so that we + # get our cookies back. + requested_uri = b"%s://%s%s" % ( + b"https" if request.isSecure() else b"http", + request.getHeader(b"host"), + request.uri, + ) + baseurl_bytes = self._public_baseurl.encode("utf-8") + if not requested_uri.startswith(baseurl_bytes): + i = requested_uri.index(b"/_matrix") + new_uri = baseurl_bytes[:-1] + requested_uri[i:] + logger.info( + "Requested URI %s is not canonical: redirecting to %s", + requested_uri.decode("utf-8", errors="replace"), + new_uri.decode("utf-8", errors="replace"), + ) + request.redirect(new_uri) + finish_request(request) + client_redirect_url = parse_string( request, "redirectUrl", required=True, encoding=None ) From 47d2b49e2b938a1c0c2e13830505a6d019ee65fe Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 18 Feb 2021 14:29:48 +0000 Subject: [PATCH 134/178] more login hacking --- synapse/http/site.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/synapse/http/site.py b/synapse/http/site.py index 4a4fb5ef26..7421c172e4 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -333,14 +333,28 @@ def _should_log_request(self) -> bool: class XForwardedForRequest(SynapseRequest): - def __init__(self, *args, **kw): - SynapseRequest.__init__(self, *args, **kw) - """ Add a layer on top of another request that only uses the value of an X-Forwarded-For header as the result of C{getClientIP}. + + XXX: I think the right way to do this is with request.setHost(). """ + def __init__(self, *args, **kw): + SynapseRequest.__init__(self, *args, **kw) + + forwarded_header = self.getHeader(b"x-forwarded-proto") + if forwarded_header is not None: + self._is_secure = forwarded_header.lower() == b"https" + else: + logger.warning( + "received request lacks an x-forwarded-proto header: assuming https" + ) + self._is_secure = True + + def isSecure(self): + return self._is_secure + def getClientIP(self): """ @return: The client address (the first address) in the value of the From c7934aee2caaed19200450cea1e94c2969b78a26 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 26 Feb 2021 14:04:05 +0000 Subject: [PATCH 135/178] Revert "more login hacking" This reverts commit 47d2b49e2b938a1c0c2e13830505a6d019ee65fe. This has now been superceded on develop by PR 9472. --- synapse/http/site.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/synapse/http/site.py b/synapse/http/site.py index 7421c172e4..4a4fb5ef26 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -333,28 +333,14 @@ def _should_log_request(self) -> bool: class XForwardedForRequest(SynapseRequest): + def __init__(self, *args, **kw): + SynapseRequest.__init__(self, *args, **kw) + """ Add a layer on top of another request that only uses the value of an X-Forwarded-For header as the result of C{getClientIP}. - - XXX: I think the right way to do this is with request.setHost(). """ - def __init__(self, *args, **kw): - SynapseRequest.__init__(self, *args, **kw) - - forwarded_header = self.getHeader(b"x-forwarded-proto") - if forwarded_header is not None: - self._is_secure = forwarded_header.lower() == b"https" - else: - logger.warning( - "received request lacks an x-forwarded-proto header: assuming https" - ) - self._is_secure = True - - def isSecure(self): - return self._is_secure - def getClientIP(self): """ @return: The client address (the first address) in the value of the From 0e56f02d5d8346f9bacd449d25c5ff7921943ef0 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 26 Feb 2021 14:05:00 +0000 Subject: [PATCH 136/178] Revert "Redirect redirect requests if they arrive on the wrong URI" This reverts commit 5ee8a1c50a1b571a8a8704a59635232193b454f2. This has now been superceded on develop by PR #9436. --- synapse/rest/client/v1/login.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 3e6a21e20f..6e2fbedd99 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -354,7 +354,6 @@ def __init__(self, hs: "HomeServer"): hs.get_oidc_handler() self._sso_handler = hs.get_sso_handler() self._msc2858_enabled = hs.config.experimental.msc2858_enabled - self._public_baseurl = hs.config.public_baseurl def register(self, http_server: HttpServer) -> None: super().register(http_server) @@ -374,28 +373,6 @@ def register(self, http_server: HttpServer) -> None: async def on_GET( self, request: SynapseRequest, idp_id: Optional[str] = None ) -> None: - if not self._public_baseurl: - raise SynapseError(400, "SSO requires a valid public_baseurl") - - # if this isn't the expected hostname, redirect to the right one, so that we - # get our cookies back. - requested_uri = b"%s://%s%s" % ( - b"https" if request.isSecure() else b"http", - request.getHeader(b"host"), - request.uri, - ) - baseurl_bytes = self._public_baseurl.encode("utf-8") - if not requested_uri.startswith(baseurl_bytes): - i = requested_uri.index(b"/_matrix") - new_uri = baseurl_bytes[:-1] + requested_uri[i:] - logger.info( - "Requested URI %s is not canonical: redirecting to %s", - requested_uri.decode("utf-8", errors="replace"), - new_uri.decode("utf-8", errors="replace"), - ) - request.redirect(new_uri) - finish_request(request) - client_redirect_url = parse_string( request, "redirectUrl", required=True, encoding=None ) From 6557eba7dc63e50e81c07805770a09ac856233b0 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 23 Mar 2021 10:24:03 +0000 Subject: [PATCH 137/178] Revert "Patch to temporarily drop cross-user m.key_share_requests (#8675)" This reverts commit d60af9305a07fadcf0270d1887c5b7d063834967. --- changelog.d/8675.misc | 1 - synapse/federation/federation_server.py | 4 ---- 2 files changed, 5 deletions(-) delete mode 100644 changelog.d/8675.misc diff --git a/changelog.d/8675.misc b/changelog.d/8675.misc deleted file mode 100644 index 7ffe38b7d9..0000000000 --- a/changelog.d/8675.misc +++ /dev/null @@ -1 +0,0 @@ -Temporarily drop cross-user m.room_key_request to_device messages over performance concerns. diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 98caf2a1a4..9839d3d016 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -955,10 +955,6 @@ async def on_edu(self, edu_type: str, origin: str, content: dict): ): return - # Temporary patch to drop cross-user key share requests - if edu_type == "m.room_key_request": - return - # Check if we have a handler on this instance handler = self.edu_handlers.get(edu_type) if handler: From 05ec9e8d37540711ef50e85e6f7bc5fa13ec8f55 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 23 Mar 2021 10:53:05 +0000 Subject: [PATCH 138/178] Revert "Patch to temporarily drop cross-user m.key_share_requests (#8675)" (#9668) We patched `matrix-org-hotfixes` a little while ago in #8675 to drop any cross-user key share requests while they were being accidentally spammed by a client. This was a temporary fix until we had some rate-limiting in place. Rate-limiting landed in https://github.com/matrix-org/synapse/pull/8957. Note that the rate-limit can't be configured, but has what appear to be [sensible defaults](https://github.com/matrix-org/synapse/blob/db2efa9c50569adbfab102b1f447f5a8312b95f3/synapse/config/ratelimiting.py#L105-L113). Note that the original patch was already actually overridden partially when the rate-limit PR landed, as they conflicted. So we've already lifted the restriction between local devices on matrix.org, but requests were still blocked from being sent over federation. This PR cleans up the remaining bits. This reverts commit d60af9305a07fadcf0270d1887c5b7d063834967. --- changelog.d/8675.misc | 1 - synapse/federation/federation_server.py | 4 ---- 2 files changed, 5 deletions(-) delete mode 100644 changelog.d/8675.misc diff --git a/changelog.d/8675.misc b/changelog.d/8675.misc deleted file mode 100644 index 7ffe38b7d9..0000000000 --- a/changelog.d/8675.misc +++ /dev/null @@ -1 +0,0 @@ -Temporarily drop cross-user m.room_key_request to_device messages over performance concerns. diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 98caf2a1a4..9839d3d016 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -955,10 +955,6 @@ async def on_edu(self, edu_type: str, origin: str, content: dict): ): return - # Temporary patch to drop cross-user key share requests - if edu_type == "m.room_key_request": - return - # Check if we have a handler on this instance handler = self.edu_handlers.get(edu_type) if handler: From edac710bc0c4dc1cd226d9ffe73a00b42c2b67d8 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 19 Apr 2021 18:56:53 +0100 Subject: [PATCH 139/178] improve efficiency of _glob_to_re --- synapse/push/push_rule_evaluator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 49ecb38522..ae077af5b5 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -230,7 +230,8 @@ def _glob_to_re(glob: str, word_boundary: bool) -> Pattern: if IS_GLOB.search(glob): r = re.escape(glob) - r = r.replace(r"\*", ".*?") + # replace 1 or more repeats of `\*` with `.*?` + r = re.sub(r"(\\\*)+", ".*?", r) r = r.replace(r"\?", ".") # handle [abc], [a-z] and [!a-z] style ranges. From 9979fef4fef1220515c4d1916c70cd461984f035 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 27 Apr 2021 13:47:39 +0100 Subject: [PATCH 140/178] Revert "Experimental Federation Speedup (#9702)" This reverts commit 05e8c70c059f8ebb066e029bc3aa3e0cefef1019. --- changelog.d/9702.misc | 1 - contrib/experiments/test_messaging.py | 42 +++--- synapse/federation/sender/__init__.py | 137 +++++++----------- .../sender/per_destination_queue.py | 15 +- .../storage/databases/main/transactions.py | 28 ++-- 5 files changed, 95 insertions(+), 128 deletions(-) delete mode 100644 changelog.d/9702.misc diff --git a/changelog.d/9702.misc b/changelog.d/9702.misc deleted file mode 100644 index c6e63450a9..0000000000 --- a/changelog.d/9702.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up federation transmission by using fewer database calls. Contributed by @ShadowJonathan. diff --git a/contrib/experiments/test_messaging.py b/contrib/experiments/test_messaging.py index 5dd172052b..31b8a68225 100644 --- a/contrib/experiments/test_messaging.py +++ b/contrib/experiments/test_messaging.py @@ -224,16 +224,14 @@ def send_message(self, room_name, sender, body): destinations = yield self.get_servers_for_context(room_name) try: - yield self.replication_layer.send_pdus( - [ - Pdu.create_new( - context=room_name, - pdu_type="sy.room.message", - content={"sender": sender, "body": body}, - origin=self.server_name, - destinations=destinations, - ) - ] + yield self.replication_layer.send_pdu( + Pdu.create_new( + context=room_name, + pdu_type="sy.room.message", + content={"sender": sender, "body": body}, + origin=self.server_name, + destinations=destinations, + ) ) except Exception as e: logger.exception(e) @@ -255,7 +253,7 @@ def join_room(self, room_name, sender, joinee): origin=self.server_name, destinations=destinations, ) - yield self.replication_layer.send_pdus([pdu]) + yield self.replication_layer.send_pdu(pdu) except Exception as e: logger.exception(e) @@ -267,18 +265,16 @@ def invite_to_room(self, room_name, sender, invitee): destinations = yield self.get_servers_for_context(room_name) try: - yield self.replication_layer.send_pdus( - [ - Pdu.create_new( - context=room_name, - is_state=True, - pdu_type="sy.room.member", - state_key=invitee, - content={"membership": "invite"}, - origin=self.server_name, - destinations=destinations, - ) - ] + yield self.replication_layer.send_pdu( + Pdu.create_new( + context=room_name, + is_state=True, + pdu_type="sy.room.member", + state_key=invitee, + content={"membership": "invite"}, + origin=self.server_name, + destinations=destinations, + ) ) except Exception as e: logger.exception(e) diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 022bbf7dad..088260c2e9 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -16,7 +16,6 @@ import logging from typing import ( TYPE_CHECKING, - Collection, Dict, Hashable, Iterable, @@ -28,12 +27,18 @@ from prometheus_client import Counter +from twisted.internet import defer + import synapse.metrics from synapse.api.presence import UserPresenceState from synapse.events import EventBase from synapse.federation.sender.per_destination_queue import PerDestinationQueue from synapse.federation.sender.transaction_manager import TransactionManager from synapse.federation.units import Edu +from synapse.logging.context import ( + make_deferred_yieldable, + run_in_background, +) from synapse.metrics import ( LaterGauge, event_processing_loop_counter, @@ -262,27 +267,15 @@ async def _process_event_queue_loop(self) -> None: if not events and next_token >= self._last_poked_id: break - async def get_destinations_for_event( - event: EventBase, - ) -> Collection[str]: - """Computes the destinations to which this event must be sent. - - This returns an empty tuple when there are no destinations to send to, - or if this event is not from this homeserver and it is not sending - it on behalf of another server. - - Will also filter out destinations which this sender is not responsible for, - if multiple federation senders exist. - """ - + async def handle_event(event: EventBase) -> None: # Only send events for this server. send_on_behalf_of = event.internal_metadata.get_send_on_behalf_of() is_mine = self.is_mine_id(event.sender) if not is_mine and send_on_behalf_of is None: - return () + return if not event.internal_metadata.should_proactively_send(): - return () + return destinations = None # type: Optional[Set[str]] if not event.prev_event_ids(): @@ -317,7 +310,7 @@ async def get_destinations_for_event( "Failed to calculate hosts in room for event: %s", event.event_id, ) - return () + return destinations = { d @@ -327,15 +320,17 @@ async def get_destinations_for_event( ) } - destinations.discard(self.server_name) - if send_on_behalf_of is not None: # If we are sending the event on behalf of another server # then it already has the event and there is no reason to # send the event to it. destinations.discard(send_on_behalf_of) + logger.debug("Sending %s to %r", event, destinations) + if destinations: + await self._send_pdu(event, destinations) + now = self.clock.time_msec() ts = await self.store.get_received_ts(event.event_id) @@ -343,29 +338,24 @@ async def get_destinations_for_event( "federation_sender" ).observe((now - ts) / 1000) - return destinations - return () - - async def get_federatable_events_and_destinations( - events: Iterable[EventBase], - ) -> List[Tuple[EventBase, Collection[str]]]: - with Measure(self.clock, "get_destinations_for_events"): - # Fetch federation destinations per event, - # skip if get_destinations_for_event returns an empty collection, - # return list of event->destinations pairs. - return [ - (event, dests) - for (event, dests) in [ - (event, await get_destinations_for_event(event)) - for event in events - ] - if dests - ] - - events_and_dests = await get_federatable_events_and_destinations(events) - - # Send corresponding events to each destination queue - await self._distribute_events(events_and_dests) + async def handle_room_events(events: Iterable[EventBase]) -> None: + with Measure(self.clock, "handle_room_events"): + for event in events: + await handle_event(event) + + events_by_room = {} # type: Dict[str, List[EventBase]] + for event in events: + events_by_room.setdefault(event.room_id, []).append(event) + + await make_deferred_yieldable( + defer.gatherResults( + [ + run_in_background(handle_room_events, evs) + for evs in events_by_room.values() + ], + consumeErrors=True, + ) + ) await self.store.update_federation_out_pos("events", next_token) @@ -383,7 +373,7 @@ async def get_federatable_events_and_destinations( events_processed_counter.inc(len(events)) event_processing_loop_room_count.labels("federation_sender").inc( - len({event.room_id for event in events}) + len(events_by_room) ) event_processing_loop_counter.labels("federation_sender").inc() @@ -395,53 +385,34 @@ async def get_federatable_events_and_destinations( finally: self._is_processing = False - async def _distribute_events( - self, - events_and_dests: Iterable[Tuple[EventBase, Collection[str]]], - ) -> None: - """Distribute events to the respective per_destination queues. - - Also persists last-seen per-room stream_ordering to 'destination_rooms'. - - Args: - events_and_dests: A list of tuples, which are (event: EventBase, destinations: Collection[str]). - Every event is paired with its intended destinations (in federation). - """ - # Tuples of room_id + destination to their max-seen stream_ordering - room_with_dest_stream_ordering = {} # type: Dict[Tuple[str, str], int] - - # List of events to send to each destination - events_by_dest = {} # type: Dict[str, List[EventBase]] + async def _send_pdu(self, pdu: EventBase, destinations: Iterable[str]) -> None: + # We loop through all destinations to see whether we already have + # a transaction in progress. If we do, stick it in the pending_pdus + # table and we'll get back to it later. - # For each event-destinations pair... - for event, destinations in events_and_dests: + destinations = set(destinations) + destinations.discard(self.server_name) + logger.debug("Sending to: %s", str(destinations)) - # (we got this from the database, it's filled) - assert event.internal_metadata.stream_ordering - - sent_pdus_destination_dist_total.inc(len(destinations)) - sent_pdus_destination_dist_count.inc() + if not destinations: + return - # ...iterate over those destinations.. - for destination in destinations: - # ...update their stream-ordering... - room_with_dest_stream_ordering[(event.room_id, destination)] = max( - event.internal_metadata.stream_ordering, - room_with_dest_stream_ordering.get((event.room_id, destination), 0), - ) + sent_pdus_destination_dist_total.inc(len(destinations)) + sent_pdus_destination_dist_count.inc() - # ...and add the event to each destination queue. - events_by_dest.setdefault(destination, []).append(event) + assert pdu.internal_metadata.stream_ordering - # Bulk-store destination_rooms stream_ids - await self.store.bulk_store_destination_rooms_entries( - room_with_dest_stream_ordering + # track the fact that we have a PDU for these destinations, + # to allow us to perform catch-up later on if the remote is unreachable + # for a while. + await self.store.store_destination_rooms_entries( + destinations, + pdu.room_id, + pdu.internal_metadata.stream_ordering, ) - for destination, pdus in events_by_dest.items(): - logger.debug("Sending %d pdus to %s", len(pdus), destination) - - self._get_per_destination_queue(destination).send_pdus(pdus) + for destination in destinations: + self._get_per_destination_queue(destination).send_pdu(pdu) async def send_read_receipt(self, receipt: ReadReceipt) -> None: """Send a RR to any other servers in the room diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index 3bb66bce32..3b053ebcfb 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -154,22 +154,19 @@ def pending_edu_count(self) -> int: + len(self._pending_edus_keyed) ) - def send_pdus(self, pdus: Iterable[EventBase]) -> None: - """Add PDUs to the queue, and start the transmission loop if necessary + def send_pdu(self, pdu: EventBase) -> None: + """Add a PDU to the queue, and start the transmission loop if necessary Args: - pdus: pdus to send + pdu: pdu to send """ if not self._catching_up or self._last_successful_stream_ordering is None: # only enqueue the PDU if we are not catching up (False) or do not # yet know if we have anything to catch up (None) - self._pending_pdus.extend(pdus) + self._pending_pdus.append(pdu) else: - self._catchup_last_skipped = max( - pdu.internal_metadata.stream_ordering - for pdu in pdus - if pdu.internal_metadata.stream_ordering is not None - ) + assert pdu.internal_metadata.stream_ordering + self._catchup_last_skipped = pdu.internal_metadata.stream_ordering self.attempt_new_transaction() diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py index b28ca61f80..82335e7a9d 100644 --- a/synapse/storage/databases/main/transactions.py +++ b/synapse/storage/databases/main/transactions.py @@ -14,7 +14,7 @@ import logging from collections import namedtuple -from typing import Dict, List, Optional, Tuple +from typing import Iterable, List, Optional, Tuple from canonicaljson import encode_canonical_json @@ -295,33 +295,37 @@ def _set_destination_retry_timings_emulated( }, ) - async def bulk_store_destination_rooms_entries( - self, room_and_destination_to_ordering: Dict[Tuple[str, str], int] - ): + async def store_destination_rooms_entries( + self, + destinations: Iterable[str], + room_id: str, + stream_ordering: int, + ) -> None: """ - Updates or creates `destination_rooms` entries for a number of events. + Updates or creates `destination_rooms` entries in batch for a single event. Args: - room_and_destination_to_ordering: A mapping of (room, destination) -> stream_id + destinations: list of destinations + room_id: the room_id of the event + stream_ordering: the stream_ordering of the event """ await self.db_pool.simple_upsert_many( table="destinations", key_names=("destination",), - key_values={(d,) for _, d in room_and_destination_to_ordering.keys()}, + key_values=[(d,) for d in destinations], value_names=[], value_values=[], desc="store_destination_rooms_entries_dests", ) + rows = [(destination, room_id) for destination in destinations] await self.db_pool.simple_upsert_many( table="destination_rooms", - key_names=("room_id", "destination"), - key_values=list(room_and_destination_to_ordering.keys()), + key_names=("destination", "room_id"), + key_values=rows, value_names=["stream_ordering"], - value_values=[ - (stream_id,) for stream_id in room_and_destination_to_ordering.values() - ], + value_values=[(stream_ordering,)] * len(rows), desc="store_destination_rooms_entries_rooms", ) From 7865bc1dfbc89dd1c238945c04ce37c24a585aa5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 May 2021 14:23:02 +0100 Subject: [PATCH 141/178] Always cache 'event_to_prev_state_group' Fixes regression in send PDU times introduced in #9905. --- changelog.d/9950.feature | 1 + synapse/handlers/message.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 changelog.d/9950.feature diff --git a/changelog.d/9950.feature b/changelog.d/9950.feature new file mode 100644 index 0000000000..96a0e7f09f --- /dev/null +++ b/changelog.d/9950.feature @@ -0,0 +1 @@ +Improve performance of sending events for worker-based deployments using Redis. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index db065ce061..798043fbf8 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1050,6 +1050,13 @@ async def cache_joined_hosts_for_event( ) if state_entry.state_group: + await self._external_cache.set( + "event_to_prev_state_group", + event.event_id, + state_entry.state_group, + expiry_ms=60 * 60 * 1000, + ) + if state_entry.state_group in self._external_cache_joined_hosts_updates: return @@ -1057,12 +1064,6 @@ async def cache_joined_hosts_for_event( # Note that the expiry times must be larger than the expiry time in # _external_cache_joined_hosts_updates. - await self._external_cache.set( - "event_to_prev_state_group", - event.event_id, - state_entry.state_group, - expiry_ms=60 * 60 * 1000, - ) await self._external_cache.set( "get_joined_hosts", str(state_entry.state_group), From 6157f02067b0053da77c6537801fa48b2edf9319 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 11 May 2021 10:49:45 +0100 Subject: [PATCH 142/178] Revert "improve efficiency of _glob_to_re" This reverts commit edac710bc0c4dc1cd226d9ffe73a00b42c2b67d8. --- synapse/push/push_rule_evaluator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index ae077af5b5..49ecb38522 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -230,8 +230,7 @@ def _glob_to_re(glob: str, word_boundary: bool) -> Pattern: if IS_GLOB.search(glob): r = re.escape(glob) - # replace 1 or more repeats of `\*` with `.*?` - r = re.sub(r"(\\\*)+", ".*?", r) + r = r.replace(r"\*", ".*?") r = r.replace(r"\?", ".") # handle [abc], [a-z] and [!a-z] style ranges. From c80e8b98de5697509a7f275f5887849e52c408bd Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 24 Jun 2021 10:57:39 +0100 Subject: [PATCH 143/178] Tweak changelog --- CHANGES.md | 33 +++++++++++++++++++-------------- changelog.d/10238.removal | 1 - 2 files changed, 19 insertions(+), 15 deletions(-) delete mode 100644 changelog.d/10238.removal diff --git a/CHANGES.md b/CHANGES.md index 3cf1814264..1fdfeef266 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,12 +1,17 @@ Synapse 1.37.0rc1 (2021-06-23) ============================== +This release deprecates the current spam checker interface. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/develop/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new generic module interface. + +This release also removes support for fetching and renewing TLS certificate using the ACME v1 protocol, which has been fully decomissioned by Let's Encrypt on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. + + Features -------- -- Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by Sorunome and anoa. ([\#6739](https://github.com/matrix-org/synapse/issues/6739), [\#9359](https://github.com/matrix-org/synapse/issues/9359), [\#10167](https://github.com/matrix-org/synapse/issues/10167), [\#10212](https://github.com/matrix-org/synapse/issues/10212), [\#10227](https://github.com/matrix-org/synapse/issues/10227)) +- Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by @Sorunome and anoa. ([\#6739](https://github.com/matrix-org/synapse/issues/6739), [\#9359](https://github.com/matrix-org/synapse/issues/9359), [\#10167](https://github.com/matrix-org/synapse/issues/10167), [\#10212](https://github.com/matrix-org/synapse/issues/10212), [\#10227](https://github.com/matrix-org/synapse/issues/10227)) - Add experimental support for backfilling history into rooms ([MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716)). ([\#9247](https://github.com/matrix-org/synapse/issues/9247)) -- Standardised the module interface. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10206](https://github.com/matrix-org/synapse/issues/10206)) +- Implement a generic interface for third-party plugin modules. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10206](https://github.com/matrix-org/synapse/issues/10206)) - Implement config option `sso.update_profile_information` to sync SSO users' profile information with the identity provider each time they login. Currently only displayname is supported. ([\#10108](https://github.com/matrix-org/synapse/issues/10108)) - Ensure that errors during startup are written to the logs and the console. ([\#10191](https://github.com/matrix-org/synapse/issues/10191)) @@ -15,13 +20,13 @@ Bugfixes -------- - Fix a bug introduced in Synapse v1.25.0 that prevented the `ip_range_whitelist` configuration option from working for federation and identity servers. Contributed by @mikure. ([\#10115](https://github.com/matrix-org/synapse/issues/10115)) -- Remove a broken import line in Synapse's admin_cmd worker. Broke in 1.33.0. ([\#10154](https://github.com/matrix-org/synapse/issues/10154)) -- Fix a bug introduced in v1.21.0 which could cause `/sync` to return immediately with an empty response. ([\#10157](https://github.com/matrix-org/synapse/issues/10157), [\#10158](https://github.com/matrix-org/synapse/issues/10158)) -- Fix a minor bug in the response to `/_matrix/client/r0/user/{user}/openid/request_token`. Contributed by @lukaslihotzki. ([\#10175](https://github.com/matrix-org/synapse/issues/10175)) +- Remove a broken import line in Synapse's `admin_cmd` worker. Broke in Synapse v1.33.0. ([\#10154](https://github.com/matrix-org/synapse/issues/10154)) +- Fix a bug introduced in Synapse v1.21.0 which could cause `/sync` to return immediately with an empty response. ([\#10157](https://github.com/matrix-org/synapse/issues/10157), [\#10158](https://github.com/matrix-org/synapse/issues/10158)) +- Fix a minor bug in the response to `/_matrix/client/r0/user/{user}/openid/request_token` causing `expires_in` to be a float instead of an integer. Contributed by @lukaslihotzki. ([\#10175](https://github.com/matrix-org/synapse/issues/10175)) - Always require users to re-authenticate for dangerous operations: deactivating an account, modifying an account password, and adding 3PIDs. ([\#10184](https://github.com/matrix-org/synapse/issues/10184)) -- Fix a bug introduced in Synpase 1.7.2 where remote server count metrics collection would be incorrectly delayed on startup. Found by @heftig. ([\#10195](https://github.com/matrix-org/synapse/issues/10195)) -- Fix a bug introduced in v1.35.1 where an `allow` key of a `m.room.join_rules` event could be applied for incorrect room versions and configurations. ([\#10208](https://github.com/matrix-org/synapse/issues/10208)) -- Fix performance regression in responding to user key requests over federation. Introduced in v1.34.0rc1. ([\#10221](https://github.com/matrix-org/synapse/issues/10221)) +- Fix a bug introduced in Synpase v1.7.2 where remote server count metrics collection would be incorrectly delayed on startup. Found by @heftig. ([\#10195](https://github.com/matrix-org/synapse/issues/10195)) +- Fix a bug introduced in Synapse v1.35.1 where an `allow` key of a `m.room.join_rules` event could be applied for incorrect room versions and configurations. ([\#10208](https://github.com/matrix-org/synapse/issues/10208)) +- Fix performance regression in responding to user key requests over federation. Introduced in Synapse v1.34.0rc1. ([\#10221](https://github.com/matrix-org/synapse/issues/10221)) Improved Documentation @@ -36,9 +41,9 @@ Improved Documentation Deprecations and Removals ------------------------- -- The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10210](https://github.com/matrix-org/synapse/issues/10210)) +- The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/develop/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. ([\#10062](https://github.com/matrix-org/synapse/issues/10062), [\#10210](https://github.com/matrix-org/synapse/issues/10210), [\#10238](https://github.com/matrix-org/synapse/issues/10238)) - Stop supporting the unstable spaces prefixes from MSC1772. ([\#10161](https://github.com/matrix-org/synapse/issues/10161)) -- Remove Synapse's support for automatically fetching and renewing certificates using the ACME v1 protocol. This protocol has been fully turned off by Let's Encrypt for existing install on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. ([\#10194](https://github.com/matrix-org/synapse/issues/10194)) +- Remove Synapse's support for automatically fetching and renewing certificates using the ACME v1 protocol. This protocol has been fully turned off by Let's Encrypt for existing installations on June 1st 2021. Admins previously using this feature should use a [reverse proxy](https://matrix-org.github.io/synapse/develop/reverse_proxy.html) to handle TLS termination, or use an external ACME client (such as [certbot](https://certbot.eff.org/)) to retrieve a certificate and key and provide them to Synapse using the `tls_certificate_path` and `tls_private_key_path` configuration settings. ([\#10194](https://github.com/matrix-org/synapse/issues/10194)) Internal Changes @@ -47,21 +52,21 @@ Internal Changes - Update the database schema versioning to support gradual migration away from legacy tables. ([\#9933](https://github.com/matrix-org/synapse/issues/9933)) - Add type hints to the federation servlets. ([\#10080](https://github.com/matrix-org/synapse/issues/10080)) - Improve OpenTracing for event persistence. ([\#10134](https://github.com/matrix-org/synapse/issues/10134), [\#10193](https://github.com/matrix-org/synapse/issues/10193)) -- Clean up the interface for injecting opentracing over HTTP. ([\#10143](https://github.com/matrix-org/synapse/issues/10143)) +- Clean up the interface for injecting OpenTracing over HTTP. ([\#10143](https://github.com/matrix-org/synapse/issues/10143)) - Limit the number of in-flight `/keys/query` requests from a single device. ([\#10144](https://github.com/matrix-org/synapse/issues/10144)) - Refactor EventPersistenceQueue. ([\#10145](https://github.com/matrix-org/synapse/issues/10145)) - Document `SYNAPSE_TEST_LOG_LEVEL` to see the logger output when running tests. ([\#10148](https://github.com/matrix-org/synapse/issues/10148)) - Update the Complement build tags in GitHub Actions to test currently experimental features. ([\#10155](https://github.com/matrix-org/synapse/issues/10155)) -- Add `synapse_federation_soft_failed_events_total` metric to track how often events are soft failed. ([\#10156](https://github.com/matrix-org/synapse/issues/10156)) +- Add a `synapse_federation_soft_failed_events_total` metric to track how often events are soft failed. ([\#10156](https://github.com/matrix-org/synapse/issues/10156)) - Fetch the corresponding complement branch when performing CI. ([\#10160](https://github.com/matrix-org/synapse/issues/10160)) - Add some developer documentation about boolean columns in database schemas. ([\#10164](https://github.com/matrix-org/synapse/issues/10164)) - Add extra logging fields to better debug where events are being soft failed. ([\#10168](https://github.com/matrix-org/synapse/issues/10168)) - Add debug logging for when we enter and exit `Measure` blocks. ([\#10183](https://github.com/matrix-org/synapse/issues/10183)) - Improve comments in structured logging code. ([\#10188](https://github.com/matrix-org/synapse/issues/10188)) -- Update MSC3083 support for modifications in the MSC. ([\#10189](https://github.com/matrix-org/synapse/issues/10189)) +- Update [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) support with modifications from the MSC. ([\#10189](https://github.com/matrix-org/synapse/issues/10189)) - Remove redundant DNS lookup limiter. ([\#10190](https://github.com/matrix-org/synapse/issues/10190)) - Upgrade `black` linting tool to 21.6b0. ([\#10197](https://github.com/matrix-org/synapse/issues/10197)) -- Expose opentracing trace id in response headers. ([\#10199](https://github.com/matrix-org/synapse/issues/10199)) +- Expose OpenTracing trace id in response headers. ([\#10199](https://github.com/matrix-org/synapse/issues/10199)) Synapse 1.36.0 (2021-06-15) diff --git a/changelog.d/10238.removal b/changelog.d/10238.removal deleted file mode 100644 index 5fb7bfb47e..0000000000 --- a/changelog.d/10238.removal +++ /dev/null @@ -1 +0,0 @@ -The current spam checker interface is deprecated in favour of a new generic modules system. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/master/UPGRADE.rst#deprecation-of-the-current-spam-checker-interface) for more information on how to update to the new system. From 5575b5cb43125d334f510027f6a3482d0b15bd4c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 28 Jun 2021 14:12:48 +0100 Subject: [PATCH 144/178] Don't send catchup transactions for matrix hq This is because there are problems with HQ atm. --- synapse/storage/databases/main/transactions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py index d211c423b2..67655fc215 100644 --- a/synapse/storage/databases/main/transactions.py +++ b/synapse/storage/databases/main/transactions.py @@ -411,6 +411,7 @@ def _get_catch_up_room_event_ids_txn( JOIN events USING (stream_ordering) WHERE destination = ? AND stream_ordering > ? + AND room_id != '!OGEhHVWSdvArJzumhm:matrix.org' ORDER BY stream_ordering LIMIT 50 """ From f271b32beaf5515eb798708f4a205f768260d8c2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 28 Jun 2021 14:17:08 +0100 Subject: [PATCH 145/178] don't send out events for matrix.org --- synapse/federation/sender/per_destination_queue.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index 3a2efd56ee..a44cc781c8 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -287,6 +287,13 @@ async def _transaction_transmission_loop(self) -> None: len(pending_pdus), ) + # Filter out HQ traffic for now + pending_pdus = [ + pdu + for pdu in pending_pdus + if pdu.room_id != "!OGEhHVWSdvArJzumhm:matrix.org" + ] + await self._transaction_manager.send_new_transaction( self._destination, pending_pdus, pending_edus ) From ffe17e47cef75fe3d2a5f4c3fbf9c65f6caee1a8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 28 Jun 2021 14:29:27 +0100 Subject: [PATCH 146/178] Fix SQL --- synapse/storage/databases/main/transactions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py index 67655fc215..b3250084c5 100644 --- a/synapse/storage/databases/main/transactions.py +++ b/synapse/storage/databases/main/transactions.py @@ -411,7 +411,7 @@ def _get_catch_up_room_event_ids_txn( JOIN events USING (stream_ordering) WHERE destination = ? AND stream_ordering > ? - AND room_id != '!OGEhHVWSdvArJzumhm:matrix.org' + AND destination_rooms.room_id != '!OGEhHVWSdvArJzumhm:matrix.org' ORDER BY stream_ordering LIMIT 50 """ From b4bdab8e52e3a236aa5869956e1f35740250f445 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 29 Jun 2021 16:13:09 +0100 Subject: [PATCH 147/178] Revert "Don't send catchup transactions for matrix hq" This reverts commits 5575b5cb43125d334f510027f6a3482d0b15bd4c, f271b32beaf5515eb798708f4a205f768260d8c2, ffe17e47cef75fe3d2a5f4c3fbf9c65f6caee1a8 --- synapse/federation/sender/per_destination_queue.py | 7 ------- synapse/storage/databases/main/transactions.py | 1 - 2 files changed, 8 deletions(-) diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index a44cc781c8..3a2efd56ee 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -287,13 +287,6 @@ async def _transaction_transmission_loop(self) -> None: len(pending_pdus), ) - # Filter out HQ traffic for now - pending_pdus = [ - pdu - for pdu in pending_pdus - if pdu.room_id != "!OGEhHVWSdvArJzumhm:matrix.org" - ] - await self._transaction_manager.send_new_transaction( self._destination, pending_pdus, pending_edus ) diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py index b3250084c5..d211c423b2 100644 --- a/synapse/storage/databases/main/transactions.py +++ b/synapse/storage/databases/main/transactions.py @@ -411,7 +411,6 @@ def _get_catch_up_room_event_ids_txn( JOIN events USING (stream_ordering) WHERE destination = ? AND stream_ordering > ? - AND destination_rooms.room_id != '!OGEhHVWSdvArJzumhm:matrix.org' ORDER BY stream_ordering LIMIT 50 """ From cfddd43bfb8e2f00e5eed1a934ad8b7467052d22 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 29 Jun 2021 20:56:46 +0100 Subject: [PATCH 148/178] bump background update rate --- synapse/storage/background_updates.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 142787fdfd..c1f4d99e19 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -84,8 +84,9 @@ class BackgroundUpdater: MINIMUM_BACKGROUND_BATCH_SIZE = 100 DEFAULT_BACKGROUND_BATCH_SIZE = 100 - BACKGROUND_UPDATE_INTERVAL_MS = 1000 - BACKGROUND_UPDATE_DURATION_MS = 100 + # temporarily increased to make stream_ordering go faster: rv 2021/06/29 + BACKGROUND_UPDATE_INTERVAL_MS = 10 + BACKGROUND_UPDATE_DURATION_MS = 1000 def __init__(self, hs: "HomeServer", database: "DatabasePool"): self._clock = hs.get_clock() From 40e92b224c827500bb9c9400d2896e572aa61ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 16 Aug 2021 13:22:38 +0200 Subject: [PATCH 149/178] Handle string read receipt data (#10606) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Handle string read receipt data Signed-off-by: Šimon Brandner * Test that we handle string read receipt data Signed-off-by: Šimon Brandner * Add changelog for #10606 Signed-off-by: Šimon Brandner * Add docs Signed-off-by: Šimon Brandner * Ignore malformed RRs Signed-off-by: Šimon Brandner * Only surround hidden = ... Signed-off-by: Šimon Brandner * Remove unnecessary argument Signed-off-by: Šimon Brandner * Update changelog.d/10606.bugfix Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> --- changelog.d/10606.bugfix | 1 + synapse/handlers/receipts.py | 9 ++++++++- tests/handlers/test_receipts.py | 23 +++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 changelog.d/10606.bugfix diff --git a/changelog.d/10606.bugfix b/changelog.d/10606.bugfix new file mode 100644 index 0000000000..bab9fd2a61 --- /dev/null +++ b/changelog.d/10606.bugfix @@ -0,0 +1 @@ +Fix errors on /sync when read receipt data is a string. Only affects homeservers with the experimental flag for [MSC2285](https://github.com/matrix-org/matrix-doc/pull/2285) enabled. Contributed by @SimonBrandner. diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index b9085bbccb..40e0c7bb1b 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -187,7 +187,14 @@ def filter_out_hidden(events: List[JsonDict], user_id: str) -> List[JsonDict]: new_users = {} for rr_user_id, user_rr in m_read.items(): - hidden = user_rr.get("hidden", None) + try: + hidden = user_rr.get("hidden") + except AttributeError: + # Due to https://github.com/matrix-org/synapse/issues/10376 + # there are cases where user_rr is a string, in those cases + # we just ignore the read receipt + continue + if hidden is not True or rr_user_id == user_id: new_users[rr_user_id] = user_rr.copy() # If hidden has a value replace hidden with the correct prefixed key diff --git a/tests/handlers/test_receipts.py b/tests/handlers/test_receipts.py index 93a9a084b2..732a12c9bd 100644 --- a/tests/handlers/test_receipts.py +++ b/tests/handlers/test_receipts.py @@ -286,6 +286,29 @@ def test_filters_out_receipt_event_with_only_hidden_receipt_and_ignores_rest(sel ], ) + def test_handles_string_data(self): + """ + Tests that an invalid shape for read-receipts is handled. + Context: https://github.com/matrix-org/synapse/issues/10603 + """ + + self._test_filters_hidden( + [ + { + "content": { + "$14356419edgd14394fHBLK:matrix.org": { + "m.read": { + "@rikj:jki.re": "string", + } + }, + }, + "room_id": "!jEsUZKDJdhlrceRyVU:example.org", + "type": "m.receipt", + }, + ], + [], + ) + def _test_filters_hidden( self, events: List[JsonDict], expected_output: List[JsonDict] ): From ac646fed06d31b151a902b1f25f6a82a2d719ffd Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 26 Aug 2021 09:45:19 +0100 Subject: [PATCH 150/178] Remove some redundant patches from the hotfixes branch --- synapse/handlers/room_list.py | 1 - synapse/handlers/sync.py | 1 - synapse/storage/background_updates.py | 5 ++--- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 8d18e07aa6..6d433fad41 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -47,7 +47,6 @@ class RoomListHandler(BaseHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) self.enable_room_list_search = hs.config.enable_room_list_search - self.response_cache: ResponseCache[ Tuple[Optional[int], Optional[str], Optional[ThirdPartyInstanceID]] ] = ResponseCache(hs.get_clock(), "room_list") diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 53ae539d74..590642f510 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -61,7 +61,6 @@ # Debug logger for https://github.com/matrix-org/synapse/issues/4422 issue4422_logger = logging.getLogger("synapse.handler.sync.4422_debug") -SYNC_RESPONSE_CACHE_MS = 2 * 60 * 1000 # Counts the number of times we returned a non-empty sync. `type` is one of # "initial_sync", "full_state_sync" or "incremental_sync", `lazy_loaded` is diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 7f975a8f16..82b31d24f1 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -84,9 +84,8 @@ class BackgroundUpdater: MINIMUM_BACKGROUND_BATCH_SIZE = 100 DEFAULT_BACKGROUND_BATCH_SIZE = 100 - # temporarily increased to make stream_ordering go faster: rv 2021/06/29 - BACKGROUND_UPDATE_INTERVAL_MS = 10 - BACKGROUND_UPDATE_DURATION_MS = 1000 + BACKGROUND_UPDATE_INTERVAL_MS = 1000 + BACKGROUND_UPDATE_DURATION_MS = 100 def __init__(self, hs: "HomeServer", database: "DatabasePool"): self._clock = hs.get_clock() From 4d03ad5255c8219b39b6f1d3e3cd032a218adb60 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Mon, 6 Sep 2021 15:57:57 +0100 Subject: [PATCH 151/178] Expand on why users should read upgrade notes --- CHANGES.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 64c30eed10..67d649a4dd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,10 @@ Synapse 1.42.0rc2 (2021-09-06) ============================== -Server administrators are reminded to read [the upgrade notes](docs/upgrade.md#upgrading-to-v1420). +This version of Synapse removes deprecated room-management admin APIs and out-of-date +email pushers, and improves error handling for fallback templates for user-interactive +authentication. For more information on these points, server administrators are +encouraged to read [the upgrade notes](docs/upgrade.md#upgrading-to-v1420). Features -------- From dffdda5f89402f4891774d3bc0ea9b6d299afc83 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 10 Sep 2021 10:42:14 +0100 Subject: [PATCH 152/178] Revert "Expand on why users should read upgrade notes" This reverts commit 4d03ad5255c8219b39b6f1d3e3cd032a218adb60. This was committed to the wrong branch - it has been superceded by ca3cb1e039 on the release branches. --- CHANGES.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 67d649a4dd..64c30eed10 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,10 +1,7 @@ Synapse 1.42.0rc2 (2021-09-06) ============================== -This version of Synapse removes deprecated room-management admin APIs and out-of-date -email pushers, and improves error handling for fallback templates for user-interactive -authentication. For more information on these points, server administrators are -encouraged to read [the upgrade notes](docs/upgrade.md#upgrading-to-v1420). +Server administrators are reminded to read [the upgrade notes](docs/upgrade.md#upgrading-to-v1420). Features -------- From c9fb203ce0fa8705b33d18f86fc77549c1e1522b Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 15 Sep 2021 14:06:21 +0100 Subject: [PATCH 153/178] Allow LruCaches to opt out of time-based expiry --- synapse/util/caches/deferred_cache.py | 2 ++ synapse/util/caches/descriptors.py | 5 +++++ synapse/util/caches/lrucache.py | 16 +++++++++++++--- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/synapse/util/caches/deferred_cache.py b/synapse/util/caches/deferred_cache.py index f05590da0d..6262efe072 100644 --- a/synapse/util/caches/deferred_cache.py +++ b/synapse/util/caches/deferred_cache.py @@ -73,6 +73,7 @@ def __init__( tree: bool = False, iterable: bool = False, apply_cache_factor_from_config: bool = True, + prune_unread_entries: bool = True, ): """ Args: @@ -105,6 +106,7 @@ def metrics_cb() -> None: size_callback=(lambda d: len(d) or 1) if iterable else None, metrics_collection_callback=metrics_cb, apply_cache_factor_from_config=apply_cache_factor_from_config, + prune_unread_entries=prune_unread_entries, ) self.thread: Optional[threading.Thread] = None diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index 1ca31e41ac..b9dcca17f1 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -258,6 +258,7 @@ def __init__( tree=False, cache_context=False, iterable=False, + prune_unread_entries: bool = True, ): super().__init__(orig, num_args=num_args, cache_context=cache_context) @@ -269,6 +270,7 @@ def __init__( self.max_entries = max_entries self.tree = tree self.iterable = iterable + self.prune_unread_entries = prune_unread_entries def __get__(self, obj, owner): cache: DeferredCache[CacheKey, Any] = DeferredCache( @@ -276,6 +278,7 @@ def __get__(self, obj, owner): max_entries=self.max_entries, tree=self.tree, iterable=self.iterable, + prune_unread_entries=self.prune_unread_entries, ) get_cache_key = self.cache_key_builder @@ -507,6 +510,7 @@ def cached( tree: bool = False, cache_context: bool = False, iterable: bool = False, + prune_unread_entries: bool = True, ) -> Callable[[F], _CachedFunction[F]]: func = lambda orig: DeferredCacheDescriptor( orig, @@ -515,6 +519,7 @@ def cached( tree=tree, cache_context=cache_context, iterable=iterable, + prune_unread_entries=prune_unread_entries, ) return cast(Callable[[F], _CachedFunction[F]], func) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 39dce9dd41..17cb98ff0b 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -202,10 +202,11 @@ def __init__( cache: "weakref.ReferenceType[LruCache]", clock: Clock, callbacks: Collection[Callable[[], None]] = (), + prune_unread_entries: bool = True, ): self._list_node = ListNode.insert_after(self, root) - self._global_list_node = None - if USE_GLOBAL_LIST: + self._global_list_node: Optional[_TimedListNode] = None + if USE_GLOBAL_LIST and prune_unread_entries: self._global_list_node = _TimedListNode.insert_after(self, GLOBAL_ROOT) self._global_list_node.update_last_access(clock) @@ -314,6 +315,7 @@ def __init__( metrics_collection_callback: Optional[Callable[[], None]] = None, apply_cache_factor_from_config: bool = True, clock: Optional[Clock] = None, + prune_unread_entries: bool = True, ): """ Args: @@ -427,7 +429,15 @@ def cache_len(): self.len = synchronized(cache_len) def add_node(key, value, callbacks: Collection[Callable[[], None]] = ()): - node = _Node(list_root, key, value, weak_ref_to_self, real_clock, callbacks) + node = _Node( + list_root, + key, + value, + weak_ref_to_self, + real_clock, + callbacks, + prune_unread_entries, + ) cache[key] = node if size_callback: From 361ffb8f0aa1cc9c5d8a0e5f0e46cbaf3b15340e Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 15 Sep 2021 14:18:42 +0100 Subject: [PATCH 154/178] Don't expire `get_users_who_share_room` & friends --- synapse/storage/databases/main/roommember.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 9beeb96aa9..a4ec6bc328 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -162,7 +162,7 @@ def _check_safe_current_state_events_membership_updated_txn(self, txn): self._check_safe_current_state_events_membership_updated_txn, ) - @cached(max_entries=100000, iterable=True) + @cached(max_entries=100000, iterable=True, prune_unread_entries=False) async def get_users_in_room(self, room_id: str) -> List[str]: return await self.db_pool.runInteraction( "get_users_in_room", self.get_users_in_room_txn, room_id @@ -439,7 +439,7 @@ async def get_local_current_membership_for_user_in_room( return results_dict.get("membership"), results_dict.get("event_id") - @cached(max_entries=500000, iterable=True) + @cached(max_entries=500000, iterable=True, prune_unread_entries=False) async def get_rooms_for_user_with_stream_ordering( self, user_id: str ) -> FrozenSet[GetRoomsForUserWithStreamOrdering]: @@ -544,7 +544,12 @@ async def get_rooms_for_user( ) return frozenset(r.room_id for r in rooms) - @cached(max_entries=500000, cache_context=True, iterable=True) + @cached( + max_entries=500000, + cache_context=True, + iterable=True, + prune_unread_entries=False, + ) async def get_users_who_share_room_with_user( self, user_id: str, cache_context: _CacheContext ) -> Set[str]: From 622785848286d614fff3fd2e2ea2d7d522bd739c Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 15 Sep 2021 14:40:26 +0100 Subject: [PATCH 155/178] Changelog --- changelog.d/10826.misc | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog.d/10826.misc diff --git a/changelog.d/10826.misc b/changelog.d/10826.misc new file mode 100644 index 0000000000..53e56fc362 --- /dev/null +++ b/changelog.d/10826.misc @@ -0,0 +1,2 @@ +Opt out of cache expiry for `get_users_who_share_room_with_user`, to hopefully improve `/sync` performance when you +haven't synced recently. From 2bb023ba2b4fa607effbbe247791551263081b72 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 27 Oct 2021 11:56:36 +0100 Subject: [PATCH 156/178] Don't seqscan event_json due to relates_to_id --- synapse/storage/databases/main/events_bg_updates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index f92d824876..ae3a8a63e4 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -1108,7 +1108,7 @@ def _event_thread_relation_txn(txn: LoggingTransaction) -> int: """ SELECT event_id, json FROM event_json LEFT JOIN event_relations USING (event_id) - WHERE event_id > ? AND relates_to_id IS NULL + WHERE event_id > ? AND event_relations.event_id IS NULL ORDER BY event_id LIMIT ? """, (last_event_id, batch_size), From 098e964bbccbe7454a7153369a8566f707c42665 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 23 Nov 2021 10:14:22 +0000 Subject: [PATCH 157/178] Prevent the media store from writing outside of the configured directory And the associated changelog/release process updates too. Applied by patch from the security fork. --- CHANGES.md | 23 +++ debian/changelog | 6 + synapse/__init__.py | 2 +- synapse/rest/media/v1/_base.py | 18 +- synapse/rest/media/v1/filepath.py | 241 +++++++++++++++++++++----- synapse/util/stringutils.py | 21 ++- tests/http/test_endpoint.py | 3 + tests/rest/media/v1/test_filepath.py | 250 +++++++++++++++++++++++++++ 8 files changed, 513 insertions(+), 51 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a435d9c592..fde8d7f81a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,26 @@ +Synapse 1.47.1 (2021-11-23) +=========================== + +This release fixes a security issue in the media store, affecting all prior releases of Synapse. Server administrators are encouraged to update Synapse as soon as possible. We are not aware of these vulnerabilities being exploited in the wild. + +Server administrators who are unable to update Synapse may use the workarounds described in the linked GitHub Security Advisory below. + +Security advisory +----------------- + +The following issue is fixed in 1.47.1. + +- **[GHSA-3hfw-x7gx-437c](https://github.com/matrix-org/synapse/security/advisories/GHSA-3hfw-x7gx-437c) / [CVE-2021-41281](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-41281): Path traversal when downloading remote media.** + + Synapse instances with the media repository enabled can be tricked into downloading a file from a remote server into an arbitrary directory, potentially outside the media store directory. + + The last two directories and file name of the path are chosen randomly by Synapse and cannot be controlled by an attacker, which limits the impact. + + Homeservers with the media repository disabled are unaffected. Homeservers configured with a federation whitelist are also unaffected. + + Fixed by [91f2bd090](https://github.com/matrix-org/synapse/commit/91f2bd090). + + Synapse 1.47.0 (2021-11-17) =========================== diff --git a/debian/changelog b/debian/changelog index ba75d0b251..35c9063388 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.47.1) stable; urgency=medium + + * New synapse release 1.47.1. + + -- Synapse Packaging team Fri, 19 Nov 2021 13:44:32 +0000 + matrix-synapse-py3 (1.47.0) stable; urgency=medium * New synapse release 1.47.0. diff --git a/synapse/__init__.py b/synapse/__init__.py index aa964afb5e..48ac38aec6 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ except ImportError: pass -__version__ = "1.47.0" +__version__ = "1.47.1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py index 014fa893d6..9b40fd8a6c 100644 --- a/synapse/rest/media/v1/_base.py +++ b/synapse/rest/media/v1/_base.py @@ -29,7 +29,7 @@ from synapse.http.server import finish_request, respond_with_json from synapse.http.site import SynapseRequest from synapse.logging.context import make_deferred_yieldable -from synapse.util.stringutils import is_ascii +from synapse.util.stringutils import is_ascii, parse_and_validate_server_name logger = logging.getLogger(__name__) @@ -51,6 +51,19 @@ def parse_media_id(request: Request) -> Tuple[str, str, Optional[str]]: + """Parses the server name, media ID and optional file name from the request URI + + Also performs some rough validation on the server name. + + Args: + request: The `Request`. + + Returns: + A tuple containing the parsed server name, media ID and optional file name. + + Raises: + SynapseError(404): if parsing or validation fail for any reason + """ try: # The type on postpath seems incorrect in Twisted 21.2.0. postpath: List[bytes] = request.postpath # type: ignore @@ -62,6 +75,9 @@ def parse_media_id(request: Request) -> Tuple[str, str, Optional[str]]: server_name = server_name_bytes.decode("utf-8") media_id = media_id_bytes.decode("utf8") + # Validate the server name, raising if invalid + parse_and_validate_server_name(server_name) + file_name = None if len(postpath) > 2: try: diff --git a/synapse/rest/media/v1/filepath.py b/synapse/rest/media/v1/filepath.py index bec77088ee..c0e15c6513 100644 --- a/synapse/rest/media/v1/filepath.py +++ b/synapse/rest/media/v1/filepath.py @@ -16,7 +16,8 @@ import functools import os import re -from typing import Any, Callable, List, TypeVar, cast +import string +from typing import Any, Callable, List, TypeVar, Union, cast NEW_FORMAT_ID_RE = re.compile(r"^\d\d\d\d-\d\d-\d\d") @@ -37,6 +38,85 @@ def _wrapped(self: "MediaFilePaths", *args: Any, **kwargs: Any) -> str: return cast(F, _wrapped) +GetPathMethod = TypeVar( + "GetPathMethod", bound=Union[Callable[..., str], Callable[..., List[str]]] +) + + +def _wrap_with_jail_check(func: GetPathMethod) -> GetPathMethod: + """Wraps a path-returning method to check that the returned path(s) do not escape + the media store directory. + + The check is not expected to ever fail, unless `func` is missing a call to + `_validate_path_component`, or `_validate_path_component` is buggy. + + Args: + func: The `MediaFilePaths` method to wrap. The method may return either a single + path, or a list of paths. Returned paths may be either absolute or relative. + + Returns: + The method, wrapped with a check to ensure that the returned path(s) lie within + the media store directory. Raises a `ValueError` if the check fails. + """ + + @functools.wraps(func) + def _wrapped( + self: "MediaFilePaths", *args: Any, **kwargs: Any + ) -> Union[str, List[str]]: + path_or_paths = func(self, *args, **kwargs) + + if isinstance(path_or_paths, list): + paths_to_check = path_or_paths + else: + paths_to_check = [path_or_paths] + + for path in paths_to_check: + # path may be an absolute or relative path, depending on the method being + # wrapped. When "appending" an absolute path, `os.path.join` discards the + # previous path, which is desired here. + normalized_path = os.path.normpath(os.path.join(self.real_base_path, path)) + if ( + os.path.commonpath([normalized_path, self.real_base_path]) + != self.real_base_path + ): + raise ValueError(f"Invalid media store path: {path!r}") + + return path_or_paths + + return cast(GetPathMethod, _wrapped) + + +ALLOWED_CHARACTERS = set( + string.ascii_letters + + string.digits + + "_-" + + ".[]:" # Domain names, IPv6 addresses and ports in server names +) +FORBIDDEN_NAMES = { + "", + os.path.curdir, # "." for the current platform + os.path.pardir, # ".." for the current platform +} + + +def _validate_path_component(name: str) -> str: + """Checks that the given string can be safely used as a path component + + Args: + name: The path component to check. + + Returns: + The path component if valid. + + Raises: + ValueError: If `name` cannot be safely used as a path component. + """ + if not ALLOWED_CHARACTERS.issuperset(name) or name in FORBIDDEN_NAMES: + raise ValueError(f"Invalid path component: {name!r}") + + return name + + class MediaFilePaths: """Describes where files are stored on disk. @@ -48,22 +128,46 @@ class MediaFilePaths: def __init__(self, primary_base_path: str): self.base_path = primary_base_path + # The media store directory, with all symlinks resolved. + self.real_base_path = os.path.realpath(primary_base_path) + + # Refuse to initialize if paths cannot be validated correctly for the current + # platform. + assert os.path.sep not in ALLOWED_CHARACTERS + assert os.path.altsep not in ALLOWED_CHARACTERS + # On Windows, paths have all sorts of weirdness which `_validate_path_component` + # does not consider. In any case, the remote media store can't work correctly + # for certain homeservers there, since ":"s aren't allowed in paths. + assert os.name == "posix" + + @_wrap_with_jail_check def local_media_filepath_rel(self, media_id: str) -> str: - return os.path.join("local_content", media_id[0:2], media_id[2:4], media_id[4:]) + return os.path.join( + "local_content", + _validate_path_component(media_id[0:2]), + _validate_path_component(media_id[2:4]), + _validate_path_component(media_id[4:]), + ) local_media_filepath = _wrap_in_base_path(local_media_filepath_rel) + @_wrap_with_jail_check def local_media_thumbnail_rel( self, media_id: str, width: int, height: int, content_type: str, method: str ) -> str: top_level_type, sub_type = content_type.split("/") file_name = "%i-%i-%s-%s-%s" % (width, height, top_level_type, sub_type, method) return os.path.join( - "local_thumbnails", media_id[0:2], media_id[2:4], media_id[4:], file_name + "local_thumbnails", + _validate_path_component(media_id[0:2]), + _validate_path_component(media_id[2:4]), + _validate_path_component(media_id[4:]), + _validate_path_component(file_name), ) local_media_thumbnail = _wrap_in_base_path(local_media_thumbnail_rel) + @_wrap_with_jail_check def local_media_thumbnail_dir(self, media_id: str) -> str: """ Retrieve the local store path of thumbnails of a given media_id @@ -76,18 +180,24 @@ def local_media_thumbnail_dir(self, media_id: str) -> str: return os.path.join( self.base_path, "local_thumbnails", - media_id[0:2], - media_id[2:4], - media_id[4:], + _validate_path_component(media_id[0:2]), + _validate_path_component(media_id[2:4]), + _validate_path_component(media_id[4:]), ) + @_wrap_with_jail_check def remote_media_filepath_rel(self, server_name: str, file_id: str) -> str: return os.path.join( - "remote_content", server_name, file_id[0:2], file_id[2:4], file_id[4:] + "remote_content", + _validate_path_component(server_name), + _validate_path_component(file_id[0:2]), + _validate_path_component(file_id[2:4]), + _validate_path_component(file_id[4:]), ) remote_media_filepath = _wrap_in_base_path(remote_media_filepath_rel) + @_wrap_with_jail_check def remote_media_thumbnail_rel( self, server_name: str, @@ -101,11 +211,11 @@ def remote_media_thumbnail_rel( file_name = "%i-%i-%s-%s-%s" % (width, height, top_level_type, sub_type, method) return os.path.join( "remote_thumbnail", - server_name, - file_id[0:2], - file_id[2:4], - file_id[4:], - file_name, + _validate_path_component(server_name), + _validate_path_component(file_id[0:2]), + _validate_path_component(file_id[2:4]), + _validate_path_component(file_id[4:]), + _validate_path_component(file_name), ) remote_media_thumbnail = _wrap_in_base_path(remote_media_thumbnail_rel) @@ -113,6 +223,7 @@ def remote_media_thumbnail_rel( # Legacy path that was used to store thumbnails previously. # Should be removed after some time, when most of the thumbnails are stored # using the new path. + @_wrap_with_jail_check def remote_media_thumbnail_rel_legacy( self, server_name: str, file_id: str, width: int, height: int, content_type: str ) -> str: @@ -120,43 +231,66 @@ def remote_media_thumbnail_rel_legacy( file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type) return os.path.join( "remote_thumbnail", - server_name, - file_id[0:2], - file_id[2:4], - file_id[4:], - file_name, + _validate_path_component(server_name), + _validate_path_component(file_id[0:2]), + _validate_path_component(file_id[2:4]), + _validate_path_component(file_id[4:]), + _validate_path_component(file_name), ) def remote_media_thumbnail_dir(self, server_name: str, file_id: str) -> str: return os.path.join( self.base_path, "remote_thumbnail", - server_name, - file_id[0:2], - file_id[2:4], - file_id[4:], + _validate_path_component(server_name), + _validate_path_component(file_id[0:2]), + _validate_path_component(file_id[2:4]), + _validate_path_component(file_id[4:]), ) + @_wrap_with_jail_check def url_cache_filepath_rel(self, media_id: str) -> str: if NEW_FORMAT_ID_RE.match(media_id): # Media id is of the form # E.g.: 2017-09-28-fsdRDt24DS234dsf - return os.path.join("url_cache", media_id[:10], media_id[11:]) + return os.path.join( + "url_cache", + _validate_path_component(media_id[:10]), + _validate_path_component(media_id[11:]), + ) else: - return os.path.join("url_cache", media_id[0:2], media_id[2:4], media_id[4:]) + return os.path.join( + "url_cache", + _validate_path_component(media_id[0:2]), + _validate_path_component(media_id[2:4]), + _validate_path_component(media_id[4:]), + ) url_cache_filepath = _wrap_in_base_path(url_cache_filepath_rel) + @_wrap_with_jail_check def url_cache_filepath_dirs_to_delete(self, media_id: str) -> List[str]: "The dirs to try and remove if we delete the media_id file" if NEW_FORMAT_ID_RE.match(media_id): - return [os.path.join(self.base_path, "url_cache", media_id[:10])] + return [ + os.path.join( + self.base_path, "url_cache", _validate_path_component(media_id[:10]) + ) + ] else: return [ - os.path.join(self.base_path, "url_cache", media_id[0:2], media_id[2:4]), - os.path.join(self.base_path, "url_cache", media_id[0:2]), + os.path.join( + self.base_path, + "url_cache", + _validate_path_component(media_id[0:2]), + _validate_path_component(media_id[2:4]), + ), + os.path.join( + self.base_path, "url_cache", _validate_path_component(media_id[0:2]) + ), ] + @_wrap_with_jail_check def url_cache_thumbnail_rel( self, media_id: str, width: int, height: int, content_type: str, method: str ) -> str: @@ -168,37 +302,46 @@ def url_cache_thumbnail_rel( if NEW_FORMAT_ID_RE.match(media_id): return os.path.join( - "url_cache_thumbnails", media_id[:10], media_id[11:], file_name + "url_cache_thumbnails", + _validate_path_component(media_id[:10]), + _validate_path_component(media_id[11:]), + _validate_path_component(file_name), ) else: return os.path.join( "url_cache_thumbnails", - media_id[0:2], - media_id[2:4], - media_id[4:], - file_name, + _validate_path_component(media_id[0:2]), + _validate_path_component(media_id[2:4]), + _validate_path_component(media_id[4:]), + _validate_path_component(file_name), ) url_cache_thumbnail = _wrap_in_base_path(url_cache_thumbnail_rel) + @_wrap_with_jail_check def url_cache_thumbnail_directory_rel(self, media_id: str) -> str: # Media id is of the form # E.g.: 2017-09-28-fsdRDt24DS234dsf if NEW_FORMAT_ID_RE.match(media_id): - return os.path.join("url_cache_thumbnails", media_id[:10], media_id[11:]) + return os.path.join( + "url_cache_thumbnails", + _validate_path_component(media_id[:10]), + _validate_path_component(media_id[11:]), + ) else: return os.path.join( "url_cache_thumbnails", - media_id[0:2], - media_id[2:4], - media_id[4:], + _validate_path_component(media_id[0:2]), + _validate_path_component(media_id[2:4]), + _validate_path_component(media_id[4:]), ) url_cache_thumbnail_directory = _wrap_in_base_path( url_cache_thumbnail_directory_rel ) + @_wrap_with_jail_check def url_cache_thumbnail_dirs_to_delete(self, media_id: str) -> List[str]: "The dirs to try and remove if we delete the media_id thumbnails" # Media id is of the form @@ -206,21 +349,35 @@ def url_cache_thumbnail_dirs_to_delete(self, media_id: str) -> List[str]: if NEW_FORMAT_ID_RE.match(media_id): return [ os.path.join( - self.base_path, "url_cache_thumbnails", media_id[:10], media_id[11:] + self.base_path, + "url_cache_thumbnails", + _validate_path_component(media_id[:10]), + _validate_path_component(media_id[11:]), + ), + os.path.join( + self.base_path, + "url_cache_thumbnails", + _validate_path_component(media_id[:10]), ), - os.path.join(self.base_path, "url_cache_thumbnails", media_id[:10]), ] else: return [ os.path.join( self.base_path, "url_cache_thumbnails", - media_id[0:2], - media_id[2:4], - media_id[4:], + _validate_path_component(media_id[0:2]), + _validate_path_component(media_id[2:4]), + _validate_path_component(media_id[4:]), ), os.path.join( - self.base_path, "url_cache_thumbnails", media_id[0:2], media_id[2:4] + self.base_path, + "url_cache_thumbnails", + _validate_path_component(media_id[0:2]), + _validate_path_component(media_id[2:4]), + ), + os.path.join( + self.base_path, + "url_cache_thumbnails", + _validate_path_component(media_id[0:2]), ), - os.path.join(self.base_path, "url_cache_thumbnails", media_id[0:2]), ] diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py index f029432191..ea1032b4fc 100644 --- a/synapse/util/stringutils.py +++ b/synapse/util/stringutils.py @@ -19,6 +19,8 @@ from collections.abc import Iterable from typing import Optional, Tuple +from netaddr import valid_ipv6 + from synapse.api.errors import Codes, SynapseError _string_with_symbols = string.digits + string.ascii_letters + ".,;:^&*-_+=#~@" @@ -97,7 +99,10 @@ def parse_server_name(server_name: str) -> Tuple[str, Optional[int]]: raise ValueError("Invalid server name '%s'" % server_name) -VALID_HOST_REGEX = re.compile("\\A[0-9a-zA-Z.-]+\\Z") +# An approximation of the domain name syntax in RFC 1035, section 2.3.1. +# NB: "\Z" is not equivalent to "$". +# The latter will match the position before a "\n" at the end of a string. +VALID_HOST_REGEX = re.compile("\\A[0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*\\Z") def parse_and_validate_server_name(server_name: str) -> Tuple[str, Optional[int]]: @@ -122,13 +127,15 @@ def parse_and_validate_server_name(server_name: str) -> Tuple[str, Optional[int] if host[0] == "[": if host[-1] != "]": raise ValueError("Mismatched [...] in server name '%s'" % (server_name,)) - return host, port - # otherwise it should only be alphanumerics. - if not VALID_HOST_REGEX.match(host): - raise ValueError( - "Server name '%s' contains invalid characters" % (server_name,) - ) + # valid_ipv6 raises when given an empty string + ipv6_address = host[1:-1] + if not ipv6_address or not valid_ipv6(ipv6_address): + raise ValueError( + "Server name '%s' is not a valid IPv6 address" % (server_name,) + ) + elif not VALID_HOST_REGEX.match(host): + raise ValueError("Server name '%s' has an invalid format" % (server_name,)) return host, port diff --git a/tests/http/test_endpoint.py b/tests/http/test_endpoint.py index 1f9a2f9b1d..c8cc21cadd 100644 --- a/tests/http/test_endpoint.py +++ b/tests/http/test_endpoint.py @@ -36,8 +36,11 @@ def test_validate_bad_server_names(self): "localhost:http", # non-numeric port "1234]", # smells like ipv6 literal but isn't "[1234", + "[1.2.3.4]", "underscore_.com", "percent%65.com", + "newline.com\n", + ".empty-label.com", "1234:5678:80", # too many colons ] for i in test_data: diff --git a/tests/rest/media/v1/test_filepath.py b/tests/rest/media/v1/test_filepath.py index 09504a485f..8fe94f7d85 100644 --- a/tests/rest/media/v1/test_filepath.py +++ b/tests/rest/media/v1/test_filepath.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import inspect +from typing import Iterable + from synapse.rest.media.v1.filepath import MediaFilePaths from tests import unittest @@ -236,3 +239,250 @@ def test_url_cache_thumbnail_dirs_to_delete_legacy(self): "/media_store/url_cache_thumbnails/Ge", ], ) + + def test_server_name_validation(self): + """Test validation of server names""" + self._test_path_validation( + [ + "remote_media_filepath_rel", + "remote_media_filepath", + "remote_media_thumbnail_rel", + "remote_media_thumbnail", + "remote_media_thumbnail_rel_legacy", + "remote_media_thumbnail_dir", + ], + parameter="server_name", + valid_values=[ + "matrix.org", + "matrix.org:8448", + "matrix-federation.matrix.org", + "matrix-federation.matrix.org:8448", + "10.1.12.123", + "10.1.12.123:8448", + "[fd00:abcd::ffff]", + "[fd00:abcd::ffff]:8448", + ], + invalid_values=[ + "/matrix.org", + "matrix.org/..", + "matrix.org\x00", + "", + ".", + "..", + "/", + ], + ) + + def test_file_id_validation(self): + """Test validation of local, remote and legacy URL cache file / media IDs""" + # File / media IDs get split into three parts to form paths, consisting of the + # first two characters, next two characters and rest of the ID. + valid_file_ids = [ + "GerZNDnDZVjsOtardLuwfIBg", + # Unexpected, but produces an acceptable path: + "GerZN", # "N" becomes the last directory + ] + invalid_file_ids = [ + "/erZNDnDZVjsOtardLuwfIBg", + "Ge/ZNDnDZVjsOtardLuwfIBg", + "GerZ/DnDZVjsOtardLuwfIBg", + "GerZ/..", + "G\x00rZNDnDZVjsOtardLuwfIBg", + "Ger\x00NDnDZVjsOtardLuwfIBg", + "GerZNDnDZVjsOtardLuwfIBg\x00", + "", + "Ge", + "GerZ", + "GerZ.", + "..rZNDnDZVjsOtardLuwfIBg", + "Ge..NDnDZVjsOtardLuwfIBg", + "GerZ..", + "GerZ/", + ] + + self._test_path_validation( + [ + "local_media_filepath_rel", + "local_media_filepath", + "local_media_thumbnail_rel", + "local_media_thumbnail", + "local_media_thumbnail_dir", + # Legacy URL cache media IDs + "url_cache_filepath_rel", + "url_cache_filepath", + # `url_cache_filepath_dirs_to_delete` is tested below. + "url_cache_thumbnail_rel", + "url_cache_thumbnail", + "url_cache_thumbnail_directory_rel", + "url_cache_thumbnail_directory", + "url_cache_thumbnail_dirs_to_delete", + ], + parameter="media_id", + valid_values=valid_file_ids, + invalid_values=invalid_file_ids, + ) + + # `url_cache_filepath_dirs_to_delete` ignores what would be the last path + # component, so only the first 4 characters matter. + self._test_path_validation( + [ + "url_cache_filepath_dirs_to_delete", + ], + parameter="media_id", + valid_values=valid_file_ids, + invalid_values=[ + "/erZNDnDZVjsOtardLuwfIBg", + "Ge/ZNDnDZVjsOtardLuwfIBg", + "G\x00rZNDnDZVjsOtardLuwfIBg", + "Ger\x00NDnDZVjsOtardLuwfIBg", + "", + "Ge", + "..rZNDnDZVjsOtardLuwfIBg", + "Ge..NDnDZVjsOtardLuwfIBg", + ], + ) + + self._test_path_validation( + [ + "remote_media_filepath_rel", + "remote_media_filepath", + "remote_media_thumbnail_rel", + "remote_media_thumbnail", + "remote_media_thumbnail_rel_legacy", + "remote_media_thumbnail_dir", + ], + parameter="file_id", + valid_values=valid_file_ids, + invalid_values=invalid_file_ids, + ) + + def test_url_cache_media_id_validation(self): + """Test validation of URL cache media IDs""" + self._test_path_validation( + [ + "url_cache_filepath_rel", + "url_cache_filepath", + # `url_cache_filepath_dirs_to_delete` only cares about the date prefix + "url_cache_thumbnail_rel", + "url_cache_thumbnail", + "url_cache_thumbnail_directory_rel", + "url_cache_thumbnail_directory", + "url_cache_thumbnail_dirs_to_delete", + ], + parameter="media_id", + valid_values=[ + "2020-01-02_GerZNDnDZVjsOtar", + "2020-01-02_G", # Unexpected, but produces an acceptable path + ], + invalid_values=[ + "2020-01-02", + "2020-01-02-", + "2020-01-02-.", + "2020-01-02-..", + "2020-01-02-/", + "2020-01-02-/GerZNDnDZVjsOtar", + "2020-01-02-GerZNDnDZVjsOtar/..", + "2020-01-02-GerZNDnDZVjsOtar\x00", + ], + ) + + def test_content_type_validation(self): + """Test validation of thumbnail content types""" + self._test_path_validation( + [ + "local_media_thumbnail_rel", + "local_media_thumbnail", + "remote_media_thumbnail_rel", + "remote_media_thumbnail", + "remote_media_thumbnail_rel_legacy", + "url_cache_thumbnail_rel", + "url_cache_thumbnail", + ], + parameter="content_type", + valid_values=[ + "image/jpeg", + ], + invalid_values=[ + "", # ValueError: not enough values to unpack + "image/jpeg/abc", # ValueError: too many values to unpack + "image/jpeg\x00", + ], + ) + + def test_thumbnail_method_validation(self): + """Test validation of thumbnail methods""" + self._test_path_validation( + [ + "local_media_thumbnail_rel", + "local_media_thumbnail", + "remote_media_thumbnail_rel", + "remote_media_thumbnail", + "url_cache_thumbnail_rel", + "url_cache_thumbnail", + ], + parameter="method", + valid_values=[ + "crop", + "scale", + ], + invalid_values=[ + "/scale", + "scale/..", + "scale\x00", + "/", + ], + ) + + def _test_path_validation( + self, + methods: Iterable[str], + parameter: str, + valid_values: Iterable[str], + invalid_values: Iterable[str], + ): + """Test that the specified methods validate the named parameter as expected + + Args: + methods: The names of `MediaFilePaths` methods to test + parameter: The name of the parameter to test + valid_values: A list of parameter values that are expected to be accepted + invalid_values: A list of parameter values that are expected to be rejected + + Raises: + AssertionError: If a value was accepted when it should have failed + validation. + ValueError: If a value failed validation when it should have been accepted. + """ + for method in methods: + get_path = getattr(self.filepaths, method) + + parameters = inspect.signature(get_path).parameters + kwargs = { + "server_name": "matrix.org", + "media_id": "GerZNDnDZVjsOtardLuwfIBg", + "file_id": "GerZNDnDZVjsOtardLuwfIBg", + "width": 800, + "height": 600, + "content_type": "image/jpeg", + "method": "scale", + } + + if get_path.__name__.startswith("url_"): + kwargs["media_id"] = "2020-01-02_GerZNDnDZVjsOtar" + + kwargs = {k: v for k, v in kwargs.items() if k in parameters} + kwargs.pop(parameter) + + for value in valid_values: + kwargs[parameter] = value + get_path(**kwargs) + # No exception should be raised + + for value in invalid_values: + with self.assertRaises(ValueError): + kwargs[parameter] = value + path_or_list = get_path(**kwargs) + self.fail( + f"{value!r} unexpectedly passed validation: " + f"{method} returned {path_or_list!r}" + ) From a9c146ac53a40d191c0a2b3b1b8f6ceaea0eee79 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 15 Dec 2021 13:08:24 +0000 Subject: [PATCH 158/178] Disable aggregation bundling on `/sync` responses A partial revert of #11478. This turns out to have had a significant CPU impact on initial-sync handling. For now, let's disable it, until we find a more efficient way of achieving this. --- changelog.d/11583.bugfix | 1 + synapse/rest/client/sync.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog.d/11583.bugfix diff --git a/changelog.d/11583.bugfix b/changelog.d/11583.bugfix new file mode 100644 index 0000000000..d2ed113e21 --- /dev/null +++ b/changelog.d/11583.bugfix @@ -0,0 +1 @@ +Fix a performance regression in `/sync` handling, introduced in 1.49.0. diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 88e4f5e063..e556ff93e6 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -522,7 +522,13 @@ def serialize(events: Iterable[EventBase]) -> Awaitable[List[JsonDict]]: time_now=time_now, # Don't bother to bundle aggregations if the timeline is unlimited, # as clients will have all the necessary information. - bundle_aggregations=room.timeline.limited, + # bundle_aggregations=room.timeline.limited, + # + # richvdh 2021-12-15: disable this temporarily as it has too high an + # overhead for initialsyncs. We need to figure out a way that the + # bundling can be done *before* the events are stored in the + # SyncResponseCache so that this part can be synchronous. + bundle_aggregations=False, token_id=token_id, event_format=event_formatter, only_event_fields=only_fields, From 99e7fb1d52204a7b7cae9f2d9e0a51e1febf8e01 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 7 Jan 2022 11:53:28 +0000 Subject: [PATCH 159/178] Wrap connection.commit with OpenTracing This is an attempt to diagnose poor apdex levels, per https://github.com/matrix-org/internal-config/issues/1181 --- synapse/storage/database.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 2cacc7dd6c..3f848ca86f 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -50,6 +50,7 @@ current_context, make_deferred_yieldable, ) +from synapse.logging.opentracing import trace from synapse.metrics import register_threadpool from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.background_updates import BackgroundUpdater @@ -104,8 +105,20 @@ def _on_new_connection(conn): # Ensure we have a logging context so we can correctly track queries, # etc. with LoggingContext("db.on_new_connection"): + # HACK Patch the connection's commit function so that we can see + # how long it's taking from Jaeger. + class NastyConnectionWrapper: + def __init__(self, connection): + self._connection = connection + self.commit = trace(connection.commit, "db.conn.commit") + + def __getattr__(self, item): + return getattr(self._connection, item) + engine.on_new_connection( - LoggingDatabaseConnection(conn, engine, "on_new_connection") + LoggingDatabaseConnection( + NastyConnectionWrapper(conn), engine, "on_new_connection" + ) ) connection_pool = adbapi.ConnectionPool( From 5cc41f1b05416954f4c9e7aea1df308f4a451abe Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 7 Jan 2022 12:49:54 +0000 Subject: [PATCH 160/178] Updates to opentracing hackery --- synapse/storage/database.py | 49 ++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 3f848ca86f..b62719e1ef 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -41,6 +41,7 @@ from typing_extensions import Literal from twisted.enterprise import adbapi +from twisted.python import reflect from synapse.api.errors import StoreError from synapse.config.database import DatabaseConnectionConfig @@ -50,7 +51,6 @@ current_context, make_deferred_yieldable, ) -from synapse.logging.opentracing import trace from synapse.metrics import register_threadpool from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.background_updates import BackgroundUpdater @@ -91,6 +91,20 @@ } +class NastyConnectionWrapper: + def __init__(self, connection): + self._connection = connection + self._synapse_parent_context = None + + def commit(self, *args, **kwargs): + with LoggingContext("db_commit", parent_context = self._synapse_parent_context): + with opentracing.start_active_span("db.conn.commit"): + self._connection.commit(*args, **kwargs) + + def __getattr__(self, item): + return getattr(self._connection, item) + + def make_pool( reactor, db_config: DatabaseConnectionConfig, engine: BaseDatabaseEngine ) -> adbapi.ConnectionPool: @@ -105,22 +119,29 @@ def _on_new_connection(conn): # Ensure we have a logging context so we can correctly track queries, # etc. with LoggingContext("db.on_new_connection"): - # HACK Patch the connection's commit function so that we can see - # how long it's taking from Jaeger. - class NastyConnectionWrapper: - def __init__(self, connection): - self._connection = connection - self.commit = trace(connection.commit, "db.conn.commit") - - def __getattr__(self, item): - return getattr(self._connection, item) - engine.on_new_connection( LoggingDatabaseConnection( - NastyConnectionWrapper(conn), engine, "on_new_connection" + conn, engine, "on_new_connection" ) ) + # HACK Patch the connection's commit function so that we can see + # how long it's taking from Jaeger. To do that, we need to patch the + # dbapi module's 'connect' method so that it returns a wrapped 'Connection' + # object to the connection pool. (psycopg2's Connection class is a C thing + # which we can't monkey-patch directly). + dbapiname = db_config.config["name"] + dbapi = reflect.namedModule(dbapiname) + if not getattr(dbapi, "_synapse_wrapped_dbapi", False): + real_connect = dbapi.connect + + def wrapped_connect(*args, **kwargs): + conn = real_connect(*args, **kwargs) + return NastyConnectionWrapper(conn) + + dbapi.connect = wrapped_connect + dbapi._synapse_wrapped_dbapi = True + connection_pool = adbapi.ConnectionPool( db_config.config["name"], cp_reactor=reactor, @@ -813,6 +834,10 @@ def inner_func(conn, *args, **kwargs): # pool). assert not self.engine.in_transaction(conn) + # HACK: record the parent context in 'conn' so that we can tie later commits + # back to it + conn._connection._synapse_parent_context = parent_context + with LoggingContext( str(curr_context), parent_context=parent_context ) as context: From 2b9f741f3a9dee93e9744774b342c35ce60062c4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sun, 23 Jan 2022 21:08:52 +0000 Subject: [PATCH 161/178] Fix logic for dropping old events in fed queue Fixes infinite loops of > logger.info("Invalid prev_events for %s", event_id) --- synapse/storage/databases/main/event_federation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 270b30800b..a556f17dac 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -1432,7 +1432,10 @@ async def prune_staged_events_in_room( if room_version.event_format == EventFormatVersions.V1: for prev_event_tuple in prev_events: - if not isinstance(prev_event_tuple, list) or len(prev_events) != 2: + if ( + not isinstance(prev_event_tuple, list) + or len(prev_event_tuple) != 2 + ): logger.info("Invalid prev_events for %s", event_id) break From 3bf466698fcc5b7038878b6549e584e1e8573f56 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Fri, 4 Feb 2022 10:54:35 +0000 Subject: [PATCH 162/178] Hotfixes: Revert `commit()` OpenTracing hackery (#11906) --- synapse/storage/database.py | 40 +------------------------------------ 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index f1d975a8ff..99802228c9 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -41,7 +41,6 @@ from typing_extensions import Literal from twisted.enterprise import adbapi -from twisted.python import reflect from synapse.api.errors import StoreError from synapse.config.database import DatabaseConnectionConfig @@ -91,20 +90,6 @@ } -class NastyConnectionWrapper: - def __init__(self, connection): - self._connection = connection - self._synapse_parent_context = None - - def commit(self, *args, **kwargs): - with LoggingContext("db_commit", parent_context = self._synapse_parent_context): - with opentracing.start_active_span("db.conn.commit"): - self._connection.commit(*args, **kwargs) - - def __getattr__(self, item): - return getattr(self._connection, item) - - def make_pool( reactor, db_config: DatabaseConnectionConfig, engine: BaseDatabaseEngine ) -> adbapi.ConnectionPool: @@ -120,28 +105,9 @@ def _on_new_connection(conn): # etc. with LoggingContext("db.on_new_connection"): engine.on_new_connection( - LoggingDatabaseConnection( - conn, engine, "on_new_connection" - ) + LoggingDatabaseConnection(conn, engine, "on_new_connection") ) - # HACK Patch the connection's commit function so that we can see - # how long it's taking from Jaeger. To do that, we need to patch the - # dbapi module's 'connect' method so that it returns a wrapped 'Connection' - # object to the connection pool. (psycopg2's Connection class is a C thing - # which we can't monkey-patch directly). - dbapiname = db_config.config["name"] - dbapi = reflect.namedModule(dbapiname) - if not getattr(dbapi, "_synapse_wrapped_dbapi", False): - real_connect = dbapi.connect - - def wrapped_connect(*args, **kwargs): - conn = real_connect(*args, **kwargs) - return NastyConnectionWrapper(conn) - - dbapi.connect = wrapped_connect - dbapi._synapse_wrapped_dbapi = True - connection_pool = adbapi.ConnectionPool( db_config.config["name"], cp_reactor=reactor, @@ -839,10 +805,6 @@ def inner_func(conn, *args, **kwargs): # pool). assert not self.engine.in_transaction(conn) - # HACK: record the parent context in 'conn' so that we can tie later commits - # back to it - conn._connection._synapse_parent_context = parent_context - with LoggingContext( str(curr_context), parent_context=parent_context ) as context: From ed2f158a380e31429759521883de95ee5cbc25bb Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 8 Feb 2022 12:57:17 +0000 Subject: [PATCH 163/178] Printf debugging for MSISDN validation (#11882) This makes some attempt to keep CI happy too, but it probably ought not to. --- synapse/handlers/message.py | 4 ++-- synapse/rest/client/account.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 4411d901ac..8f44af2d92 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -277,8 +277,8 @@ async def get_joined_members(self, requester: Requester, room_id: str) -> dict: # If this is an AS, double check that they are allowed to see the members. # This can either be because the AS user is in the room or because there # is a user in the room that the AS is "interested in" - if False and requester.app_service and user_id not in users_with_profile: - for uid in users_with_profile: + if False and requester.app_service and user_id not in users_with_profile: # type: ignore[unreachable] + for uid in users_with_profile: # type: ignore[unreachable] if requester.app_service.is_interested_in_user(uid): break else: diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index 6b272658fc..d9cb55e35b 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -467,6 +467,7 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: next_link = body.get("next_link") # Optional param msisdn = phone_number_to_msisdn(country, phone_number) + logger.info("Request #%s to verify ownership of %s", send_attempt, msisdn) if not check_3pid_allowed(self.hs, "msisdn", msisdn): raise SynapseError( @@ -494,6 +495,7 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: await self.hs.get_clock().sleep(random.randint(1, 10) / 10) return 200, {"sid": random_string(16)} + logger.info("MSISDN %s is already in use by %s", msisdn, existing_user_id) raise SynapseError(400, "MSISDN is already in use", Codes.THREEPID_IN_USE) if not self.hs.config.registration.account_threepid_delegate_msisdn: @@ -518,6 +520,7 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: threepid_send_requests.labels(type="msisdn", reason="add_threepid").observe( send_attempt ) + logger.info("MSISDN %s: got response from identity server: %s", msisdn, ret) return 200, ret From f1b625ad56d539305e64f529bcfd25915e5d1be5 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 1 Mar 2022 15:30:22 +0000 Subject: [PATCH 164/178] matrix.org hotfixes: Back out in-flight state cache changes (#12117) --- changelog.d/10870.misc | 1 - changelog.d/11608.misc | 1 - synapse/storage/databases/state/store.py | 219 ++---------- tests/storage/databases/test_state_store.py | 352 -------------------- 4 files changed, 25 insertions(+), 548 deletions(-) delete mode 100644 changelog.d/10870.misc delete mode 100644 changelog.d/11608.misc delete mode 100644 tests/storage/databases/test_state_store.py diff --git a/changelog.d/10870.misc b/changelog.d/10870.misc deleted file mode 100644 index 3af049b969..0000000000 --- a/changelog.d/10870.misc +++ /dev/null @@ -1 +0,0 @@ -Deduplicate in-flight requests in `_get_state_for_groups`. diff --git a/changelog.d/11608.misc b/changelog.d/11608.misc deleted file mode 100644 index 3af049b969..0000000000 --- a/changelog.d/11608.misc +++ /dev/null @@ -1 +0,0 @@ -Deduplicate in-flight requests in `_get_state_for_groups`. diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index b8016f679a..7614d76ac6 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -13,23 +13,11 @@ # limitations under the License. import logging -from typing import ( - TYPE_CHECKING, - Collection, - Dict, - Iterable, - Optional, - Sequence, - Set, - Tuple, -) +from typing import TYPE_CHECKING, Collection, Dict, Iterable, List, Optional, Set, Tuple import attr -from twisted.internet import defer - from synapse.api.constants import EventTypes -from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.storage._base import SQLBaseStore from synapse.storage.database import ( DatabasePool, @@ -41,12 +29,6 @@ from synapse.storage.types import Cursor from synapse.storage.util.sequence import build_sequence_generator from synapse.types import MutableStateMap, StateKey, StateMap -from synapse.util import unwrapFirstError -from synapse.util.async_helpers import ( - AbstractObservableDeferred, - ObservableDeferred, - yieldable_gather_results, -) from synapse.util.caches.descriptors import cached from synapse.util.caches.dictionary_cache import DictionaryCache @@ -55,8 +37,8 @@ logger = logging.getLogger(__name__) + MAX_STATE_DELTA_HOPS = 100 -MAX_INFLIGHT_REQUESTS_PER_GROUP = 5 @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -124,12 +106,6 @@ def __init__( 500000, ) - # Current ongoing get_state_for_groups in-flight requests - # {group ID -> {StateFilter -> ObservableDeferred}} - self._state_group_inflight_requests: Dict[ - int, Dict[StateFilter, AbstractObservableDeferred[StateMap[str]]] - ] = {} - def get_max_state_group_txn(txn: Cursor) -> int: txn.execute("SELECT COALESCE(max(id), 0) FROM state_groups") return txn.fetchone()[0] # type: ignore @@ -181,7 +157,7 @@ def _get_state_group_delta_txn(txn: LoggingTransaction) -> _GetStateGroupDelta: ) async def _get_state_groups_from_groups( - self, groups: Sequence[int], state_filter: StateFilter + self, groups: List[int], state_filter: StateFilter ) -> Dict[int, StateMap[str]]: """Returns the state groups for a given set of groups from the database, filtering on types of state events. @@ -252,165 +228,6 @@ def _get_state_for_group_using_cache( return state_filter.filter_state(state_dict_ids), not missing_types - def _get_state_for_group_gather_inflight_requests( - self, group: int, state_filter_left_over: StateFilter - ) -> Tuple[Sequence[AbstractObservableDeferred[StateMap[str]]], StateFilter]: - """ - Attempts to gather in-flight requests and re-use them to retrieve state - for the given state group, filtered with the given state filter. - - If there are more than MAX_INFLIGHT_REQUESTS_PER_GROUP in-flight requests, - and there *still* isn't enough information to complete the request by solely - reusing others, a full state filter will be requested to ensure that subsequent - requests can reuse this request. - - Used as part of _get_state_for_group_using_inflight_cache. - - Returns: - Tuple of two values: - A sequence of ObservableDeferreds to observe - A StateFilter representing what else needs to be requested to fulfill the request - """ - - inflight_requests = self._state_group_inflight_requests.get(group) - if inflight_requests is None: - # no requests for this group, need to retrieve it all ourselves - return (), state_filter_left_over - - # The list of ongoing requests which will help narrow the current request. - reusable_requests = [] - for (request_state_filter, request_deferred) in inflight_requests.items(): - new_state_filter_left_over = state_filter_left_over.approx_difference( - request_state_filter - ) - if new_state_filter_left_over == state_filter_left_over: - # Reusing this request would not gain us anything, so don't bother. - continue - - reusable_requests.append(request_deferred) - state_filter_left_over = new_state_filter_left_over - if state_filter_left_over == StateFilter.none(): - # we have managed to collect enough of the in-flight requests - # to cover our StateFilter and give us the state we need. - break - - if ( - state_filter_left_over != StateFilter.none() - and len(inflight_requests) >= MAX_INFLIGHT_REQUESTS_PER_GROUP - ): - # There are too many requests for this group. - # To prevent even more from building up, we request the whole - # state filter to guarantee that we can be reused by any subsequent - # requests for this state group. - return (), StateFilter.all() - - return reusable_requests, state_filter_left_over - - async def _get_state_for_group_fire_request( - self, group: int, state_filter: StateFilter - ) -> StateMap[str]: - """ - Fires off a request to get the state at a state group, - potentially filtering by type and/or state key. - - This request will be tracked in the in-flight request cache and automatically - removed when it is finished. - - Used as part of _get_state_for_group_using_inflight_cache. - - Args: - group: ID of the state group for which we want to get state - state_filter: the state filter used to fetch state from the database - """ - cache_sequence_nm = self._state_group_cache.sequence - cache_sequence_m = self._state_group_members_cache.sequence - - # Help the cache hit ratio by expanding the filter a bit - db_state_filter = state_filter.return_expanded() - - async def _the_request() -> StateMap[str]: - group_to_state_dict = await self._get_state_groups_from_groups( - (group,), state_filter=db_state_filter - ) - - # Now let's update the caches - self._insert_into_cache( - group_to_state_dict, - db_state_filter, - cache_seq_num_members=cache_sequence_m, - cache_seq_num_non_members=cache_sequence_nm, - ) - - # Remove ourselves from the in-flight cache - group_request_dict = self._state_group_inflight_requests[group] - del group_request_dict[db_state_filter] - if not group_request_dict: - # If there are no more requests in-flight for this group, - # clean up the cache by removing the empty dictionary - del self._state_group_inflight_requests[group] - - return group_to_state_dict[group] - - # We don't immediately await the result, so must use run_in_background - # But we DO await the result before the current log context (request) - # finishes, so don't need to run it as a background process. - request_deferred = run_in_background(_the_request) - observable_deferred = ObservableDeferred(request_deferred, consumeErrors=True) - - # Insert the ObservableDeferred into the cache - group_request_dict = self._state_group_inflight_requests.setdefault(group, {}) - group_request_dict[db_state_filter] = observable_deferred - - return await make_deferred_yieldable(observable_deferred.observe()) - - async def _get_state_for_group_using_inflight_cache( - self, group: int, state_filter: StateFilter - ) -> MutableStateMap[str]: - """ - Gets the state at a state group, potentially filtering by type and/or - state key. - - 1. Calls _get_state_for_group_gather_inflight_requests to gather any - ongoing requests which might overlap with the current request. - 2. Fires a new request, using _get_state_for_group_fire_request, - for any state which cannot be gathered from ongoing requests. - - Args: - group: ID of the state group for which we want to get state - state_filter: the state filter used to fetch state from the database - Returns: - state map - """ - - # first, figure out whether we can re-use any in-flight requests - # (and if so, what would be left over) - ( - reusable_requests, - state_filter_left_over, - ) = self._get_state_for_group_gather_inflight_requests(group, state_filter) - - if state_filter_left_over != StateFilter.none(): - # Fetch remaining state - remaining = await self._get_state_for_group_fire_request( - group, state_filter_left_over - ) - assembled_state: MutableStateMap[str] = dict(remaining) - else: - assembled_state = {} - - gathered = await make_deferred_yieldable( - defer.gatherResults( - (r.observe() for r in reusable_requests), consumeErrors=True - ) - ).addErrback(unwrapFirstError) - - # assemble our result. - for result_piece in gathered: - assembled_state.update(result_piece) - - # Filter out any state that may be more than what we asked for. - return state_filter.filter_state(assembled_state) - async def _get_state_for_groups( self, groups: Iterable[int], state_filter: Optional[StateFilter] = None ) -> Dict[int, MutableStateMap[str]]: @@ -452,17 +269,31 @@ async def _get_state_for_groups( if not incomplete_groups: return state - async def get_from_cache(group: int, state_filter: StateFilter) -> None: - state[group] = await self._get_state_for_group_using_inflight_cache( - group, state_filter - ) + cache_sequence_nm = self._state_group_cache.sequence + cache_sequence_m = self._state_group_members_cache.sequence - await yieldable_gather_results( - get_from_cache, - incomplete_groups, - state_filter, + # Help the cache hit ratio by expanding the filter a bit + db_state_filter = state_filter.return_expanded() + + group_to_state_dict = await self._get_state_groups_from_groups( + list(incomplete_groups), state_filter=db_state_filter ) + # Now lets update the caches + self._insert_into_cache( + group_to_state_dict, + db_state_filter, + cache_seq_num_members=cache_sequence_m, + cache_seq_num_non_members=cache_sequence_nm, + ) + + # And finally update the result dict, by filtering out any extra + # stuff we pulled out of the database. + for group, group_state_dict in group_to_state_dict.items(): + # We just replace any existing entries, as we will have loaded + # everything we need from the database anyway. + state[group] = state_filter.filter_state(group_state_dict) + return state def _get_state_for_groups_using_cache( diff --git a/tests/storage/databases/test_state_store.py b/tests/storage/databases/test_state_store.py deleted file mode 100644 index 076b660809..0000000000 --- a/tests/storage/databases/test_state_store.py +++ /dev/null @@ -1,352 +0,0 @@ -# Copyright 2022 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import typing -from typing import Dict, List, Sequence, Tuple -from unittest.mock import patch - -from twisted.internet.defer import Deferred, ensureDeferred -from twisted.test.proto_helpers import MemoryReactor - -from synapse.api.constants import EventTypes -from synapse.storage.databases.state.store import MAX_INFLIGHT_REQUESTS_PER_GROUP -from synapse.storage.state import StateFilter -from synapse.types import StateMap -from synapse.util import Clock - -from tests.unittest import HomeserverTestCase - -if typing.TYPE_CHECKING: - from synapse.server import HomeServer - -# StateFilter for ALL non-m.room.member state events -ALL_NON_MEMBERS_STATE_FILTER = StateFilter.freeze( - types={EventTypes.Member: set()}, - include_others=True, -) - -FAKE_STATE = { - (EventTypes.Member, "@alice:test"): "join", - (EventTypes.Member, "@bob:test"): "leave", - (EventTypes.Member, "@charlie:test"): "invite", - ("test.type", "a"): "AAA", - ("test.type", "b"): "BBB", - ("other.event.type", "state.key"): "123", -} - - -class StateGroupInflightCachingTestCase(HomeserverTestCase): - def prepare( - self, reactor: MemoryReactor, clock: Clock, homeserver: "HomeServer" - ) -> None: - self.state_storage = homeserver.get_storage().state - self.state_datastore = homeserver.get_datastores().state - # Patch out the `_get_state_groups_from_groups`. - # This is useful because it lets us pretend we have a slow database. - get_state_groups_patch = patch.object( - self.state_datastore, - "_get_state_groups_from_groups", - self._fake_get_state_groups_from_groups, - ) - get_state_groups_patch.start() - - self.addCleanup(get_state_groups_patch.stop) - self.get_state_group_calls: List[ - Tuple[Tuple[int, ...], StateFilter, Deferred[Dict[int, StateMap[str]]]] - ] = [] - - def _fake_get_state_groups_from_groups( - self, groups: Sequence[int], state_filter: StateFilter - ) -> "Deferred[Dict[int, StateMap[str]]]": - d: Deferred[Dict[int, StateMap[str]]] = Deferred() - self.get_state_group_calls.append((tuple(groups), state_filter, d)) - return d - - def _complete_request_fake( - self, - groups: Tuple[int, ...], - state_filter: StateFilter, - d: "Deferred[Dict[int, StateMap[str]]]", - ) -> None: - """ - Assemble a fake database response and complete the database request. - """ - - # Return a filtered copy of the fake state - d.callback({group: state_filter.filter_state(FAKE_STATE) for group in groups}) - - def test_duplicate_requests_deduplicated(self) -> None: - """ - Tests that duplicate requests for state are deduplicated. - - This test: - - requests some state (state group 42, 'all' state filter) - - requests it again, before the first request finishes - - checks to see that only one database query was made - - completes the database query - - checks that both requests see the same retrieved state - """ - req1 = ensureDeferred( - self.state_datastore._get_state_for_group_using_inflight_cache( - 42, StateFilter.all() - ) - ) - self.pump(by=0.1) - - # This should have gone to the database - self.assertEqual(len(self.get_state_group_calls), 1) - self.assertFalse(req1.called) - - req2 = ensureDeferred( - self.state_datastore._get_state_for_group_using_inflight_cache( - 42, StateFilter.all() - ) - ) - self.pump(by=0.1) - - # No more calls should have gone to the database - self.assertEqual(len(self.get_state_group_calls), 1) - self.assertFalse(req1.called) - self.assertFalse(req2.called) - - groups, sf, d = self.get_state_group_calls[0] - self.assertEqual(groups, (42,)) - self.assertEqual(sf, StateFilter.all()) - - # Now we can complete the request - self._complete_request_fake(groups, sf, d) - - self.assertEqual(self.get_success(req1), FAKE_STATE) - self.assertEqual(self.get_success(req2), FAKE_STATE) - - def test_smaller_request_deduplicated(self) -> None: - """ - Tests that duplicate requests for state are deduplicated. - - This test: - - requests some state (state group 42, 'all' state filter) - - requests a subset of that state, before the first request finishes - - checks to see that only one database query was made - - completes the database query - - checks that both requests see the correct retrieved state - """ - req1 = ensureDeferred( - self.state_datastore._get_state_for_group_using_inflight_cache( - 42, StateFilter.from_types((("test.type", None),)) - ) - ) - self.pump(by=0.1) - - # This should have gone to the database - self.assertEqual(len(self.get_state_group_calls), 1) - self.assertFalse(req1.called) - - req2 = ensureDeferred( - self.state_datastore._get_state_for_group_using_inflight_cache( - 42, StateFilter.from_types((("test.type", "b"),)) - ) - ) - self.pump(by=0.1) - - # No more calls should have gone to the database, because the second - # request was already in the in-flight cache! - self.assertEqual(len(self.get_state_group_calls), 1) - self.assertFalse(req1.called) - self.assertFalse(req2.called) - - groups, sf, d = self.get_state_group_calls[0] - self.assertEqual(groups, (42,)) - # The state filter is expanded internally for increased cache hit rate, - # so we the database sees a wider state filter than requested. - self.assertEqual(sf, ALL_NON_MEMBERS_STATE_FILTER) - - # Now we can complete the request - self._complete_request_fake(groups, sf, d) - - self.assertEqual( - self.get_success(req1), - {("test.type", "a"): "AAA", ("test.type", "b"): "BBB"}, - ) - self.assertEqual(self.get_success(req2), {("test.type", "b"): "BBB"}) - - def test_partially_overlapping_request_deduplicated(self) -> None: - """ - Tests that partially-overlapping requests are partially deduplicated. - - This test: - - requests a single type of wildcard state - (This is internally expanded to be all non-member state) - - requests the entire state in parallel - - checks to see that two database queries were made, but that the second - one is only for member state. - - completes the database queries - - checks that both requests have the correct result. - """ - - req1 = ensureDeferred( - self.state_datastore._get_state_for_group_using_inflight_cache( - 42, StateFilter.from_types((("test.type", None),)) - ) - ) - self.pump(by=0.1) - - # This should have gone to the database - self.assertEqual(len(self.get_state_group_calls), 1) - self.assertFalse(req1.called) - - req2 = ensureDeferred( - self.state_datastore._get_state_for_group_using_inflight_cache( - 42, StateFilter.all() - ) - ) - self.pump(by=0.1) - - # Because it only partially overlaps, this also went to the database - self.assertEqual(len(self.get_state_group_calls), 2) - self.assertFalse(req1.called) - self.assertFalse(req2.called) - - # First request: - groups, sf, d = self.get_state_group_calls[0] - self.assertEqual(groups, (42,)) - # The state filter is expanded internally for increased cache hit rate, - # so we the database sees a wider state filter than requested. - self.assertEqual(sf, ALL_NON_MEMBERS_STATE_FILTER) - self._complete_request_fake(groups, sf, d) - - # Second request: - groups, sf, d = self.get_state_group_calls[1] - self.assertEqual(groups, (42,)) - # The state filter is narrowed to only request membership state, because - # the remainder of the state is already being queried in the first request! - self.assertEqual( - sf, StateFilter.freeze({EventTypes.Member: None}, include_others=False) - ) - self._complete_request_fake(groups, sf, d) - - # Check the results are correct - self.assertEqual( - self.get_success(req1), - {("test.type", "a"): "AAA", ("test.type", "b"): "BBB"}, - ) - self.assertEqual(self.get_success(req2), FAKE_STATE) - - def test_in_flight_requests_stop_being_in_flight(self) -> None: - """ - Tests that in-flight request deduplication doesn't somehow 'hold on' - to completed requests: once they're done, they're taken out of the - in-flight cache. - """ - req1 = ensureDeferred( - self.state_datastore._get_state_for_group_using_inflight_cache( - 42, StateFilter.all() - ) - ) - self.pump(by=0.1) - - # This should have gone to the database - self.assertEqual(len(self.get_state_group_calls), 1) - self.assertFalse(req1.called) - - # Complete the request right away. - self._complete_request_fake(*self.get_state_group_calls[0]) - self.assertTrue(req1.called) - - # Send off another request - req2 = ensureDeferred( - self.state_datastore._get_state_for_group_using_inflight_cache( - 42, StateFilter.all() - ) - ) - self.pump(by=0.1) - - # It should have gone to the database again, because the previous request - # isn't in-flight and therefore isn't available for deduplication. - self.assertEqual(len(self.get_state_group_calls), 2) - self.assertFalse(req2.called) - - # Complete the request right away. - self._complete_request_fake(*self.get_state_group_calls[1]) - self.assertTrue(req2.called) - groups, sf, d = self.get_state_group_calls[0] - - self.assertEqual(self.get_success(req1), FAKE_STATE) - self.assertEqual(self.get_success(req2), FAKE_STATE) - - def test_inflight_requests_capped(self) -> None: - """ - Tests that the number of in-flight requests is capped to 5. - - - requests several pieces of state separately - (5 to hit the limit, 1 to 'shunt out', another that comes after the - group has been 'shunted out') - - checks to see that the torrent of requests is shunted out by - rewriting one of the filters as the 'all' state filter - - requests after that one do not cause any additional queries - """ - # 5 at the time of writing. - CAP_COUNT = MAX_INFLIGHT_REQUESTS_PER_GROUP - - reqs = [] - - # Request 7 different keys (1 to 7) of the `some.state` type. - for req_id in range(CAP_COUNT + 2): - reqs.append( - ensureDeferred( - self.state_datastore._get_state_for_group_using_inflight_cache( - 42, - StateFilter.freeze( - {"some.state": {str(req_id + 1)}}, include_others=False - ), - ) - ) - ) - self.pump(by=0.1) - - # There should only be 6 calls to the database, not 7. - self.assertEqual(len(self.get_state_group_calls), CAP_COUNT + 1) - - # Assert that the first 5 are exact requests for the individual pieces - # wanted - for req_id in range(CAP_COUNT): - groups, sf, d = self.get_state_group_calls[req_id] - self.assertEqual( - sf, - StateFilter.freeze( - {"some.state": {str(req_id + 1)}}, include_others=False - ), - ) - - # The 6th request should be the 'all' state filter - groups, sf, d = self.get_state_group_calls[CAP_COUNT] - self.assertEqual(sf, StateFilter.all()) - - # Complete the queries and check which requests complete as a result - for req_id in range(CAP_COUNT): - # This request should not have been completed yet - self.assertFalse(reqs[req_id].called) - - groups, sf, d = self.get_state_group_calls[req_id] - self._complete_request_fake(groups, sf, d) - - # This should have only completed this one request - self.assertTrue(reqs[req_id].called) - - # Now complete the final query; the last 2 requests should complete - # as a result - self.assertFalse(reqs[CAP_COUNT].called) - self.assertFalse(reqs[CAP_COUNT + 1].called) - groups, sf, d = self.get_state_group_calls[CAP_COUNT] - self._complete_request_fake(groups, sf, d) - self.assertTrue(reqs[CAP_COUNT].called) - self.assertTrue(reqs[CAP_COUNT + 1].called) From 36383d48f3efa01039898dac294ecce0aa2d172e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 4 Apr 2022 19:07:49 +0100 Subject: [PATCH 165/178] Prefill the device_list_stream_cache --- synapse/replication/slave/storage/devices.py | 12 +++++++++++- synapse/storage/databases/main/__init__.py | 12 +++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/synapse/replication/slave/storage/devices.py b/synapse/replication/slave/storage/devices.py index 0ffd34f1da..a0dd9d2b89 100644 --- a/synapse/replication/slave/storage/devices.py +++ b/synapse/replication/slave/storage/devices.py @@ -47,8 +47,18 @@ def __init__( ], ) device_list_max = self._device_list_id_gen.get_current_token() + device_list_prefill, min_device_list_id = self.db_pool.get_cache_dict( + db_conn, + "device_lists_stream", + entity_column="user_id", + stream_column="stream_id", + max_value=device_list_max, + limit=1000, + ) self._device_list_stream_cache = StreamChangeCache( - "DeviceListStreamChangeCache", device_list_max + "DeviceListStreamChangeCache", + min_device_list_id, + prefilled_cache=device_list_prefill, ) self._user_signature_stream_cache = StreamChangeCache( "UserSignatureStreamChangeCache", device_list_max diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py index f024761ba7..68abf6783f 100644 --- a/synapse/storage/databases/main/__init__.py +++ b/synapse/storage/databases/main/__init__.py @@ -183,8 +183,18 @@ def __init__( super().__init__(database, db_conn, hs) device_list_max = self._device_list_id_gen.get_current_token() + device_list_prefill, min_device_list_id = self.db_pool.get_cache_dict( + db_conn, + "device_lists_stream", + entity_column="user_id", + stream_column="stream_id", + max_value=device_list_max, + limit=1000, + ) self._device_list_stream_cache = StreamChangeCache( - "DeviceListStreamChangeCache", device_list_max + "DeviceListStreamChangeCache", + min_device_list_id, + prefilled_cache=device_list_prefill, ) self._user_signature_stream_cache = StreamChangeCache( "UserSignatureStreamChangeCache", device_list_max From f986db2adb6f1e97a51299fb5af6abfa49e24f19 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 4 Apr 2022 19:08:42 +0100 Subject: [PATCH 166/178] Newsfile --- changelog.d/12367.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/12367.feature diff --git a/changelog.d/12367.feature b/changelog.d/12367.feature new file mode 100644 index 0000000000..89403ceeef --- /dev/null +++ b/changelog.d/12367.feature @@ -0,0 +1 @@ +Reduce overhead of restarting synchrotrons From ea2d01366ce2a68c11b0ed8f21d90a6596463215 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 4 Apr 2022 19:13:58 +0100 Subject: [PATCH 167/178] Newsfile --- changelog.d/12367.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/12367.feature b/changelog.d/12367.feature index 89403ceeef..34bb60e966 100644 --- a/changelog.d/12367.feature +++ b/changelog.d/12367.feature @@ -1 +1 @@ -Reduce overhead of restarting synchrotrons +Reduce overhead of restarting synchrotrons. From 771d2041cd02e4f16bd21d0f8580806911648926 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 May 2022 15:43:00 +0100 Subject: [PATCH 168/178] Fix query performance for /sync Broke in #12729. --- synapse/storage/databases/main/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 3c3137fe64..0373af86c8 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -750,7 +750,7 @@ def _f(txn: LoggingTransaction) -> Optional[Tuple[int, int, str]]: WHERE room_id = ? AND stream_ordering <= ? AND NOT outlier - AND rejections.reason IS NULL + AND rejections.event_id IS NULL ORDER BY stream_ordering DESC LIMIT 1 """ From 4cee2a6330c572117b56246bae8571ccc5445ee9 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Wed, 18 May 2022 12:23:37 +0100 Subject: [PATCH 169/178] Fixup changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index e10ac0314a..9bb2357572 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,7 @@ Synapse 1.59.1 (2022-05-18) =========================== -This release fixes a long-standing issue which could prevent Synapse's user directory for updating properly. +This release fixes a long-standing issue which could prevent Synapse's user directory from updating properly. Bugfixes ---------------- From 7ba03360109e3868ddd79df825d76cdd7120720d Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 23 Aug 2022 11:31:19 +0100 Subject: [PATCH 170/178] Remove the hotfix-specific hack for auth on room_members This reverts fae708c0e8c35930f1172322b7c0e9f0b1b3f9a4. We believe this to be unnecessary---other Synapse deployments do not have this patch, and we are not aware of bridging problems as a result. Related: - https://github.com/matrix-org/matrix-appservice-irc/issues/506 - https://github.com/matrix-org/synapse/issues/4826 --- synapse/handlers/message.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 9aea02b6b9..acd3de06f6 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -339,9 +339,7 @@ async def get_joined_members(self, requester: Requester, room_id: str) -> dict: # This can either be because the AS user is in the room or because there # is a user in the room that the AS is "interested in" if ( - False # See https://github.com/matrix-org/matrix-appservice-irc/issues/506 - # and https://github.com/matrix-org/synapse/issues/4826#issuecomment-1028105662 - and requester.app_service + requester.app_service and requester.user.to_string() not in users_with_profile ): for uid in users_with_profile: From 220af1df54c3b4e5f1d7ec8fb62b375c99a2bbe2 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 2 Nov 2022 12:28:24 +0000 Subject: [PATCH 171/178] Revert "Switch search SQL to triple-quote strings. (#14311)" This reverts commit 81815e0561eea91dbf0c29731589fac2e6f98a40. --- synapse/storage/databases/main/search.py | 188 +++++++++++------------ 1 file changed, 89 insertions(+), 99 deletions(-) diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index e9588d1755..594b935614 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -80,11 +80,11 @@ def store_search_entries_txn( if not self.hs.config.server.enable_search: return if isinstance(self.database_engine, PostgresEngine): - sql = """ - INSERT INTO event_search - (event_id, room_id, key, vector, stream_ordering, origin_server_ts) - VALUES (?,?,?,to_tsvector('english', ?),?,?) - """ + sql = ( + "INSERT INTO event_search" + " (event_id, room_id, key, vector, stream_ordering, origin_server_ts)" + " VALUES (?,?,?,to_tsvector('english', ?),?,?)" + ) args1 = ( ( @@ -101,20 +101,20 @@ def store_search_entries_txn( txn.execute_batch(sql, args1) elif isinstance(self.database_engine, Sqlite3Engine): - self.db_pool.simple_insert_many_txn( - txn, - table="event_search", - keys=("event_id", "room_id", "key", "value"), - values=( - ( - entry.event_id, - entry.room_id, - entry.key, - _clean_value_for_search(entry.value), - ) - for entry in entries - ), + sql = ( + "INSERT INTO event_search (event_id, room_id, key, value)" + " VALUES (?,?,?,?)" + ) + args2 = ( + ( + entry.event_id, + entry.room_id, + entry.key, + _clean_value_for_search(entry.value), + ) + for entry in entries ) + txn.execute_batch(sql, args2) else: # This should be unreachable. @@ -162,17 +162,15 @@ async def _background_reindex_search( TYPES = ["m.room.name", "m.room.message", "m.room.topic"] def reindex_search_txn(txn: LoggingTransaction) -> int: - sql = """ - SELECT stream_ordering, event_id, room_id, type, json, origin_server_ts - FROM events - JOIN event_json USING (room_id, event_id) - WHERE ? <= stream_ordering AND stream_ordering < ? - AND (%s) - ORDER BY stream_ordering DESC - LIMIT ? - """ % ( - " OR ".join("type = '%s'" % (t,) for t in TYPES), - ) + sql = ( + "SELECT stream_ordering, event_id, room_id, type, json, " + " origin_server_ts FROM events" + " JOIN event_json USING (room_id, event_id)" + " WHERE ? <= stream_ordering AND stream_ordering < ?" + " AND (%s)" + " ORDER BY stream_ordering DESC" + " LIMIT ?" + ) % (" OR ".join("type = '%s'" % (t,) for t in TYPES),) txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size)) @@ -286,10 +284,8 @@ def create_index(conn: LoggingDatabaseConnection) -> None: try: c.execute( - """ - CREATE INDEX CONCURRENTLY event_search_fts_idx - ON event_search USING GIN (vector) - """ + "CREATE INDEX CONCURRENTLY event_search_fts_idx" + " ON event_search USING GIN (vector)" ) except psycopg2.ProgrammingError as e: logger.warning( @@ -327,16 +323,12 @@ def create_index(conn: LoggingDatabaseConnection) -> None: # We create with NULLS FIRST so that when we search *backwards* # we get the ones with non null origin_server_ts *first* c.execute( - """ - CREATE INDEX CONCURRENTLY event_search_room_order - ON event_search(room_id, origin_server_ts NULLS FIRST, stream_ordering NULLS FIRST) - """ + "CREATE INDEX CONCURRENTLY event_search_room_order ON event_search(" + "room_id, origin_server_ts NULLS FIRST, stream_ordering NULLS FIRST)" ) c.execute( - """ - CREATE INDEX CONCURRENTLY event_search_order - ON event_search(origin_server_ts NULLS FIRST, stream_ordering NULLS FIRST) - """ + "CREATE INDEX CONCURRENTLY event_search_order ON event_search(" + "origin_server_ts NULLS FIRST, stream_ordering NULLS FIRST)" ) conn.set_session(autocommit=False) @@ -353,14 +345,14 @@ def create_index(conn: LoggingDatabaseConnection) -> None: ) def reindex_search_txn(txn: LoggingTransaction) -> Tuple[int, bool]: - sql = """ - UPDATE event_search AS es - SET stream_ordering = e.stream_ordering, origin_server_ts = e.origin_server_ts - FROM events AS e - WHERE e.event_id = es.event_id - AND ? <= e.stream_ordering AND e.stream_ordering < ? - RETURNING es.stream_ordering - """ + sql = ( + "UPDATE event_search AS es SET stream_ordering = e.stream_ordering," + " origin_server_ts = e.origin_server_ts" + " FROM events AS e" + " WHERE e.event_id = es.event_id" + " AND ? <= e.stream_ordering AND e.stream_ordering < ?" + " RETURNING es.stream_ordering" + ) min_stream_id = max_stream_id - batch_size txn.execute(sql, (min_stream_id, max_stream_id)) @@ -464,33 +456,33 @@ async def search_msgs( if isinstance(self.database_engine, PostgresEngine): search_query = search_term tsquery_func = self.database_engine.tsquery_func - sql = f""" - SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) AS rank, - room_id, event_id - FROM event_search - WHERE vector @@ {tsquery_func}('english', ?) - """ + sql = ( + f"SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) AS rank," + " room_id, event_id" + " FROM event_search" + f" WHERE vector @@ {tsquery_func}('english', ?)" + ) args = [search_query, search_query] + args - count_sql = f""" - SELECT room_id, count(*) as count FROM event_search - WHERE vector @@ {tsquery_func}('english', ?) - """ + count_sql = ( + "SELECT room_id, count(*) as count FROM event_search" + f" WHERE vector @@ {tsquery_func}('english', ?)" + ) count_args = [search_query] + count_args elif isinstance(self.database_engine, Sqlite3Engine): search_query = _parse_query_for_sqlite(search_term) - sql = """ - SELECT rank(matchinfo(event_search)) as rank, room_id, event_id - FROM event_search - WHERE value MATCH ? - """ + sql = ( + "SELECT rank(matchinfo(event_search)) as rank, room_id, event_id" + " FROM event_search" + " WHERE value MATCH ?" + ) args = [search_query] + args - count_sql = """ - SELECT room_id, count(*) as count FROM event_search - WHERE value MATCH ? - """ + count_sql = ( + "SELECT room_id, count(*) as count FROM event_search" + " WHERE value MATCH ?" + ) count_args = [search_query] + count_args else: # This should be unreachable. @@ -596,27 +588,26 @@ async def search_rooms( raise SynapseError(400, "Invalid pagination token") clauses.append( - """ - (origin_server_ts < ? OR (origin_server_ts = ? AND stream_ordering < ?)) - """ + "(origin_server_ts < ?" + " OR (origin_server_ts = ? AND stream_ordering < ?))" ) args.extend([origin_server_ts, origin_server_ts, stream]) if isinstance(self.database_engine, PostgresEngine): search_query = search_term tsquery_func = self.database_engine.tsquery_func - sql = f""" - SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) as rank, - origin_server_ts, stream_ordering, room_id, event_id - FROM event_search - WHERE vector @@ {tsquery_func}('english', ?) AND - """ + sql = ( + f"SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) as rank," + " origin_server_ts, stream_ordering, room_id, event_id" + " FROM event_search" + f" WHERE vector @@ {tsquery_func}('english', ?) AND " + ) args = [search_query, search_query] + args - count_sql = f""" - SELECT room_id, count(*) as count FROM event_search - WHERE vector @@ {tsquery_func}('english', ?) AND - """ + count_sql = ( + "SELECT room_id, count(*) as count FROM event_search" + f" WHERE vector @@ {tsquery_func}('english', ?) AND " + ) count_args = [search_query] + count_args elif isinstance(self.database_engine, Sqlite3Engine): @@ -628,24 +619,23 @@ async def search_rooms( # in the events table to get the topological ordering. We need # to use the indexes in this order because sqlite refuses to # MATCH unless it uses the full text search index - sql = """ - SELECT - rank(matchinfo) as rank, room_id, event_id, origin_server_ts, stream_ordering - FROM ( - SELECT key, event_id, matchinfo(event_search) as matchinfo - FROM event_search - WHERE value MATCH ? + sql = ( + "SELECT rank(matchinfo) as rank, room_id, event_id," + " origin_server_ts, stream_ordering" + " FROM (SELECT key, event_id, matchinfo(event_search) as matchinfo" + " FROM event_search" + " WHERE value MATCH ?" + " )" + " CROSS JOIN events USING (event_id)" + " WHERE " ) - CROSS JOIN events USING (event_id) - WHERE - """ search_query = _parse_query_for_sqlite(search_term) args = [search_query] + args - count_sql = """ - SELECT room_id, count(*) as count FROM event_search - WHERE value MATCH ? AND - """ + count_sql = ( + "SELECT room_id, count(*) as count FROM event_search" + " WHERE value MATCH ? AND " + ) count_args = [search_query] + count_args else: # This should be unreachable. @@ -657,10 +647,10 @@ async def search_rooms( # We add an arbitrary limit here to ensure we don't try to pull the # entire table from the database. if isinstance(self.database_engine, PostgresEngine): - sql += """ - ORDER BY origin_server_ts DESC NULLS LAST, stream_ordering DESC NULLS LAST - LIMIT ? - """ + sql += ( + " ORDER BY origin_server_ts DESC NULLS LAST," + " stream_ordering DESC NULLS LAST LIMIT ?" + ) elif isinstance(self.database_engine, Sqlite3Engine): sql += " ORDER BY origin_server_ts DESC, stream_ordering DESC LIMIT ?" else: From 7f77f1386c94c753755b79d7d7025e958b228282 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 2 Nov 2022 12:28:47 +0000 Subject: [PATCH 172/178] Revert "Fix tests for change in PostgreSQL 14 behavior change. (#14310)" This reverts commit 67583281e3f8ea923eedbc56a4c85c7ba75d1582. --- synapse/storage/databases/main/search.py | 5 +++-- tests/storage/test_room_search.py | 16 ++++------------ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index 594b935614..a89fc54c2c 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -824,8 +824,9 @@ def _tokenize_query(query: str) -> TokenList: in_phrase = False parts = deque(query.split('"')) for i, part in enumerate(parts): - # The contents inside double quotes is treated as a phrase. - in_phrase = bool(i % 2) + # The contents inside double quotes is treated as a phrase, a trailing + # double quote is not implied. + in_phrase = bool(i % 2) and i != (len(parts) - 1) # Pull out the individual words, discarding any non-word characters. words = deque(re.findall(r"([\w\-]+)", part, re.UNICODE)) diff --git a/tests/storage/test_room_search.py b/tests/storage/test_room_search.py index 868b5bee84..9ddc19900a 100644 --- a/tests/storage/test_room_search.py +++ b/tests/storage/test_room_search.py @@ -239,6 +239,7 @@ class MessageSearchTest(HomeserverTestCase): ("fox -nope", (True, False)), ("fox -brown", (False, True)), ('"fox" quick', True), + ('"fox quick', True), ('"quick brown', True), ('" quick "', True), ('" nope"', False), @@ -268,15 +269,6 @@ def prepare( response = self.helper.send(self.room_id, self.PHRASE, tok=self.access_token) self.assertIn("event_id", response) - # The behaviour of a missing trailing double quote changed in PostgreSQL 14 - # from ignoring the initial double quote to treating it as a phrase. - main_store = homeserver.get_datastores().main - found = False - if isinstance(main_store.database_engine, PostgresEngine): - assert main_store.database_engine._version is not None - found = main_store.database_engine._version < 140000 - self.COMMON_CASES.append(('"fox quick', (found, True))) - def test_tokenize_query(self) -> None: """Test the custom logic to tokenize a user's query.""" cases = ( @@ -288,9 +280,9 @@ def test_tokenize_query(self) -> None: ("fox -brown", ["fox", SearchToken.Not, "brown"]), ("- fox", [SearchToken.Not, "fox"]), ('"fox" quick', [Phrase(["fox"]), SearchToken.And, "quick"]), - # No trailing double quote. - ('"fox quick', [Phrase(["fox", "quick"])]), - ('"-fox quick', [Phrase(["-fox", "quick"])]), + # No trailing double quoe. + ('"fox quick', ["fox", SearchToken.And, "quick"]), + ('"-fox quick', [SearchToken.Not, "fox", SearchToken.And, "quick"]), ('" quick "', [Phrase(["quick"])]), ( 'q"uick brow"n', From 7e0dd52782806592df61a1a517660b694515dc27 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 2 Nov 2022 12:29:34 +0000 Subject: [PATCH 173/178] Revert "Unified search query syntax using the full-text search capabilities of the underlying DB. (#11635)" This reverts commit d902181de98399d90c46c4e4e2cf631064757941. --- synapse/storage/databases/main/search.py | 197 +++------------- synapse/storage/engines/postgres.py | 16 -- .../73/10_update_sqlite_fts4_tokenizer.py | 62 ----- tests/storage/test_room_search.py | 213 ------------------ 4 files changed, 35 insertions(+), 453 deletions(-) delete mode 100644 synapse/storage/schema/main/delta/73/10_update_sqlite_fts4_tokenizer.py diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index a89fc54c2c..1b79acf955 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -11,22 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import enum + import logging import re -from collections import deque -from dataclasses import dataclass -from typing import ( - TYPE_CHECKING, - Any, - Collection, - Iterable, - List, - Optional, - Set, - Tuple, - Union, -) +from typing import TYPE_CHECKING, Any, Collection, Iterable, List, Optional, Set, Tuple import attr @@ -39,7 +27,7 @@ LoggingTransaction, ) from synapse.storage.databases.main.events_worker import EventRedactBehaviour -from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine from synapse.types import JsonDict if TYPE_CHECKING: @@ -433,6 +421,8 @@ async def search_msgs( """ clauses = [] + search_query = _parse_query(self.database_engine, search_term) + args: List[Any] = [] # Make sure we don't explode because the person is in too many rooms. @@ -454,24 +444,20 @@ async def search_msgs( count_clauses = clauses if isinstance(self.database_engine, PostgresEngine): - search_query = search_term - tsquery_func = self.database_engine.tsquery_func sql = ( - f"SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) AS rank," + "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) AS rank," " room_id, event_id" " FROM event_search" - f" WHERE vector @@ {tsquery_func}('english', ?)" + " WHERE vector @@ to_tsquery('english', ?)" ) args = [search_query, search_query] + args count_sql = ( "SELECT room_id, count(*) as count FROM event_search" - f" WHERE vector @@ {tsquery_func}('english', ?)" + " WHERE vector @@ to_tsquery('english', ?)" ) count_args = [search_query] + count_args elif isinstance(self.database_engine, Sqlite3Engine): - search_query = _parse_query_for_sqlite(search_term) - sql = ( "SELECT rank(matchinfo(event_search)) as rank, room_id, event_id" " FROM event_search" @@ -483,7 +469,7 @@ async def search_msgs( "SELECT room_id, count(*) as count FROM event_search" " WHERE value MATCH ?" ) - count_args = [search_query] + count_args + count_args = [search_term] + count_args else: # This should be unreachable. raise Exception("Unrecognized database engine") @@ -515,9 +501,7 @@ async def search_msgs( highlights = None if isinstance(self.database_engine, PostgresEngine): - highlights = await self._find_highlights_in_postgres( - search_query, events, tsquery_func - ) + highlights = await self._find_highlights_in_postgres(search_query, events) count_sql += " GROUP BY room_id" @@ -526,6 +510,7 @@ async def search_msgs( ) count = sum(row["count"] for row in count_results if row["room_id"] in room_ids) + return { "results": [ {"event": event_map[r["event_id"]], "rank": r["rank"]} @@ -557,6 +542,9 @@ async def search_rooms( Each match as a dictionary. """ clauses = [] + + search_query = _parse_query(self.database_engine, search_term) + args: List[Any] = [] # Make sure we don't explode because the person is in too many rooms. @@ -594,23 +582,20 @@ async def search_rooms( args.extend([origin_server_ts, origin_server_ts, stream]) if isinstance(self.database_engine, PostgresEngine): - search_query = search_term - tsquery_func = self.database_engine.tsquery_func sql = ( - f"SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) as rank," + "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) as rank," " origin_server_ts, stream_ordering, room_id, event_id" " FROM event_search" - f" WHERE vector @@ {tsquery_func}('english', ?) AND " + " WHERE vector @@ to_tsquery('english', ?) AND " ) args = [search_query, search_query] + args count_sql = ( "SELECT room_id, count(*) as count FROM event_search" - f" WHERE vector @@ {tsquery_func}('english', ?) AND " + " WHERE vector @@ to_tsquery('english', ?) AND " ) count_args = [search_query] + count_args elif isinstance(self.database_engine, Sqlite3Engine): - # We use CROSS JOIN here to ensure we use the right indexes. # https://sqlite.org/optoverview.html#crossjoin # @@ -629,14 +614,13 @@ async def search_rooms( " CROSS JOIN events USING (event_id)" " WHERE " ) - search_query = _parse_query_for_sqlite(search_term) args = [search_query] + args count_sql = ( "SELECT room_id, count(*) as count FROM event_search" " WHERE value MATCH ? AND " ) - count_args = [search_query] + count_args + count_args = [search_term] + count_args else: # This should be unreachable. raise Exception("Unrecognized database engine") @@ -676,9 +660,7 @@ async def search_rooms( highlights = None if isinstance(self.database_engine, PostgresEngine): - highlights = await self._find_highlights_in_postgres( - search_query, events, tsquery_func - ) + highlights = await self._find_highlights_in_postgres(search_query, events) count_sql += " GROUP BY room_id" @@ -704,7 +686,7 @@ async def search_rooms( } async def _find_highlights_in_postgres( - self, search_query: str, events: List[EventBase], tsquery_func: str + self, search_query: str, events: List[EventBase] ) -> Set[str]: """Given a list of events and a search term, return a list of words that match from the content of the event. @@ -715,7 +697,6 @@ async def _find_highlights_in_postgres( Args: search_query events: A list of events - tsquery_func: The tsquery_* function to use when making queries Returns: A set of strings. @@ -748,7 +729,7 @@ def f(txn: LoggingTransaction) -> Set[str]: while stop_sel in value: stop_sel += ">" - query = f"SELECT ts_headline(?, {tsquery_func}('english', ?), %s)" % ( + query = "SELECT ts_headline(?, to_tsquery('english', ?), %s)" % ( _to_postgres_options( { "StartSel": start_sel, @@ -779,128 +760,20 @@ def _to_postgres_options(options_dict: JsonDict) -> str: return "'%s'" % (",".join("%s=%s" % (k, v) for k, v in options_dict.items()),) -@dataclass -class Phrase: - phrase: List[str] - - -class SearchToken(enum.Enum): - Not = enum.auto() - Or = enum.auto() - And = enum.auto() - - -Token = Union[str, Phrase, SearchToken] -TokenList = List[Token] - - -def _is_stop_word(word: str) -> bool: - # TODO Pull these out of the dictionary: - # https://github.com/postgres/postgres/blob/master/src/backend/snowball/stopwords/english.stop - return word in {"the", "a", "you", "me", "and", "but"} - - -def _tokenize_query(query: str) -> TokenList: - """ - Convert the user-supplied `query` into a TokenList, which can be translated into - some DB-specific syntax. - - The following constructs are supported: - - - phrase queries using "double quotes" - - case-insensitive `or` and `and` operators - - negation of a keyword via unary `-` - - unary hyphen to denote NOT e.g. 'include -exclude' - - The following differs from websearch_to_tsquery: - - - Stop words are not removed. - - Unclosed phrases are treated differently. - - """ - tokens: TokenList = [] - - # Find phrases. - in_phrase = False - parts = deque(query.split('"')) - for i, part in enumerate(parts): - # The contents inside double quotes is treated as a phrase, a trailing - # double quote is not implied. - in_phrase = bool(i % 2) and i != (len(parts) - 1) - - # Pull out the individual words, discarding any non-word characters. - words = deque(re.findall(r"([\w\-]+)", part, re.UNICODE)) - - # Phrases have simplified handling of words. - if in_phrase: - # Skip stop words. - phrase = [word for word in words if not _is_stop_word(word)] - - # Consecutive words are implicitly ANDed together. - if tokens and tokens[-1] not in (SearchToken.Not, SearchToken.Or): - tokens.append(SearchToken.And) - - # Add the phrase. - tokens.append(Phrase(phrase)) - continue - - # Otherwise, not in a phrase. - while words: - word = words.popleft() - - if word.startswith("-"): - tokens.append(SearchToken.Not) - - # If there's more word, put it back to be processed again. - word = word[1:] - if word: - words.appendleft(word) - elif word.lower() == "or": - tokens.append(SearchToken.Or) - else: - # Skip stop words. - if _is_stop_word(word): - continue - - # Consecutive words are implicitly ANDed together. - if tokens and tokens[-1] not in (SearchToken.Not, SearchToken.Or): - tokens.append(SearchToken.And) - - # Add the search term. - tokens.append(word) - - return tokens - - -def _tokens_to_sqlite_match_query(tokens: TokenList) -> str: - """ - Convert the list of tokens to a string suitable for passing to sqlite's MATCH. - Assume sqlite was compiled with enhanced query syntax. - - Ref: https://www.sqlite.org/fts3.html#full_text_index_queries +def _parse_query(database_engine: BaseDatabaseEngine, search_term: str) -> str: + """Takes a plain unicode string from the user and converts it into a form + that can be passed to database. + We use this so that we can add prefix matching, which isn't something + that is supported by default. """ - match_query = [] - for token in tokens: - if isinstance(token, str): - match_query.append(token) - elif isinstance(token, Phrase): - match_query.append('"' + " ".join(token.phrase) + '"') - elif token == SearchToken.Not: - # TODO: SQLite treats NOT as a *binary* operator. Hopefully a search - # term has already been added before this. - match_query.append(" NOT ") - elif token == SearchToken.Or: - match_query.append(" OR ") - elif token == SearchToken.And: - match_query.append(" AND ") - else: - raise ValueError(f"unknown token {token}") - - return "".join(match_query) + # Pull out the individual words, discarding any non-word characters. + results = re.findall(r"([\w\-]+)", search_term, re.UNICODE) -def _parse_query_for_sqlite(search_term: str) -> str: - """Takes a plain unicode string from the user and converts it into a form - that can be passed to sqllite's matchinfo(). - """ - return _tokens_to_sqlite_match_query(_tokenize_query(search_term)) + if isinstance(database_engine, PostgresEngine): + return " & ".join(result + ":*" for result in results) + elif isinstance(database_engine, Sqlite3Engine): + return " & ".join(result + "*" for result in results) + else: + # This should be unreachable. + raise Exception("Unrecognized database engine") diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 9bf74bbf59..d8c0f64d9a 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -170,22 +170,6 @@ def supports_returning(self) -> bool: """Do we support the `RETURNING` clause in insert/update/delete?""" return True - @property - def tsquery_func(self) -> str: - """ - Selects a tsquery_* func to use. - - Ref: https://www.postgresql.org/docs/current/textsearch-controls.html - - Returns: - The function name. - """ - # Postgres 11 added support for websearch_to_tsquery. - assert self._version is not None - if self._version >= 110000: - return "websearch_to_tsquery" - return "plainto_tsquery" - def is_deadlock(self, error: Exception) -> bool: if isinstance(error, psycopg2.DatabaseError): # https://www.postgresql.org/docs/current/static/errcodes-appendix.html diff --git a/synapse/storage/schema/main/delta/73/10_update_sqlite_fts4_tokenizer.py b/synapse/storage/schema/main/delta/73/10_update_sqlite_fts4_tokenizer.py deleted file mode 100644 index 3de0a709eb..0000000000 --- a/synapse/storage/schema/main/delta/73/10_update_sqlite_fts4_tokenizer.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2022 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json - -from synapse.storage.engines import BaseDatabaseEngine, Sqlite3Engine -from synapse.storage.types import Cursor - - -def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None: - """ - Upgrade the event_search table to use the porter tokenizer if it isn't already - - Applies only for sqlite. - """ - if not isinstance(database_engine, Sqlite3Engine): - return - - # Rebuild the table event_search table with tokenize=porter configured. - cur.execute("DROP TABLE event_search") - cur.execute( - """ - CREATE VIRTUAL TABLE event_search - USING fts4 (tokenize=porter, event_id, room_id, sender, key, value ) - """ - ) - - # Re-run the background job to re-populate the event_search table. - cur.execute("SELECT MIN(stream_ordering) FROM events") - row = cur.fetchone() - min_stream_id = row[0] - - # If there are not any events, nothing to do. - if min_stream_id is None: - return - - cur.execute("SELECT MAX(stream_ordering) FROM events") - row = cur.fetchone() - max_stream_id = row[0] - - progress = { - "target_min_stream_id_inclusive": min_stream_id, - "max_stream_id_exclusive": max_stream_id + 1, - } - progress_json = json.dumps(progress) - - sql = """ - INSERT into background_updates (ordering, update_name, progress_json) - VALUES (?, ?, ?) - """ - - cur.execute(sql, (7310, "event_search", progress_json)) diff --git a/tests/storage/test_room_search.py b/tests/storage/test_room_search.py index 9ddc19900a..e747c6b50e 100644 --- a/tests/storage/test_room_search.py +++ b/tests/storage/test_room_search.py @@ -12,22 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Tuple, Union -from unittest.case import SkipTest -from unittest.mock import PropertyMock, patch - -from twisted.test.proto_helpers import MemoryReactor - import synapse.rest.admin from synapse.api.constants import EventTypes from synapse.api.errors import StoreError from synapse.rest.client import login, room -from synapse.server import HomeServer -from synapse.storage.databases.main import DataStore -from synapse.storage.databases.main.search import Phrase, SearchToken, _tokenize_query from synapse.storage.engines import PostgresEngine -from synapse.storage.engines.sqlite import Sqlite3Engine -from synapse.util import Clock from tests.unittest import HomeserverTestCase, skip_unless from tests.utils import USE_POSTGRES_FOR_TESTS @@ -198,205 +187,3 @@ def test_sqlite_non_string_deletion_background_update(self): ), ) self.assertCountEqual(values, ["hi", "2"]) - - -class MessageSearchTest(HomeserverTestCase): - """ - Check message search. - - A powerful way to check the behaviour is to run the following in Postgres >= 11: - - # SELECT websearch_to_tsquery('english', ); - - The result can be compared to the tokenized version for SQLite and Postgres < 11. - - """ - - servlets = [ - synapse.rest.admin.register_servlets_for_client_rest_resource, - login.register_servlets, - room.register_servlets, - ] - - PHRASE = "the quick brown fox jumps over the lazy dog" - - # Each entry is a search query, followed by either a boolean of whether it is - # in the phrase OR a tuple of booleans: whether it matches using websearch - # and using plain search. - COMMON_CASES: List[Tuple[str, Union[bool, Tuple[bool, bool]]]] = [ - ("nope", False), - ("brown", True), - ("quick brown", True), - ("brown quick", True), - ("quick \t brown", True), - ("jump", True), - ("brown nope", False), - ('"brown quick"', (False, True)), - ('"jumps over"', True), - ('"quick fox"', (False, True)), - ("nope OR doublenope", False), - ("furphy OR fox", (True, False)), - ("fox -nope", (True, False)), - ("fox -brown", (False, True)), - ('"fox" quick', True), - ('"fox quick', True), - ('"quick brown', True), - ('" quick "', True), - ('" nope"', False), - ] - # TODO Test non-ASCII cases. - - # Case that fail on SQLite. - POSTGRES_CASES: List[Tuple[str, Union[bool, Tuple[bool, bool]]]] = [ - # SQLite treats NOT as a binary operator. - ("- fox", (False, True)), - ("- nope", (True, False)), - ('"-fox quick', (False, True)), - # PostgreSQL skips stop words. - ('"the quick brown"', True), - ('"over lazy"', True), - ] - - def prepare( - self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer - ) -> None: - # Register a user and create a room, create some messages - self.register_user("alice", "password") - self.access_token = self.login("alice", "password") - self.room_id = self.helper.create_room_as("alice", tok=self.access_token) - - # Send the phrase as a message and check it was created - response = self.helper.send(self.room_id, self.PHRASE, tok=self.access_token) - self.assertIn("event_id", response) - - def test_tokenize_query(self) -> None: - """Test the custom logic to tokenize a user's query.""" - cases = ( - ("brown", ["brown"]), - ("quick brown", ["quick", SearchToken.And, "brown"]), - ("quick \t brown", ["quick", SearchToken.And, "brown"]), - ('"brown quick"', [Phrase(["brown", "quick"])]), - ("furphy OR fox", ["furphy", SearchToken.Or, "fox"]), - ("fox -brown", ["fox", SearchToken.Not, "brown"]), - ("- fox", [SearchToken.Not, "fox"]), - ('"fox" quick', [Phrase(["fox"]), SearchToken.And, "quick"]), - # No trailing double quoe. - ('"fox quick', ["fox", SearchToken.And, "quick"]), - ('"-fox quick', [SearchToken.Not, "fox", SearchToken.And, "quick"]), - ('" quick "', [Phrase(["quick"])]), - ( - 'q"uick brow"n', - [ - "q", - SearchToken.And, - Phrase(["uick", "brow"]), - SearchToken.And, - "n", - ], - ), - ( - '-"quick brown"', - [SearchToken.Not, Phrase(["quick", "brown"])], - ), - ) - - for query, expected in cases: - tokenized = _tokenize_query(query) - self.assertEqual( - tokenized, expected, f"{tokenized} != {expected} for {query}" - ) - - def _check_test_cases( - self, - store: DataStore, - cases: List[Tuple[str, Union[bool, Tuple[bool, bool]]]], - index=0, - ) -> None: - # Run all the test cases versus search_msgs - for query, expect_to_contain in cases: - if isinstance(expect_to_contain, tuple): - expect_to_contain = expect_to_contain[index] - - result = self.get_success( - store.search_msgs([self.room_id], query, ["content.body"]) - ) - self.assertEquals( - result["count"], - 1 if expect_to_contain else 0, - f"expected '{query}' to match '{self.PHRASE}'" - if expect_to_contain - else f"'{query}' unexpectedly matched '{self.PHRASE}'", - ) - self.assertEquals( - len(result["results"]), - 1 if expect_to_contain else 0, - "results array length should match count", - ) - - # Run them again versus search_rooms - for query, expect_to_contain in cases: - if isinstance(expect_to_contain, tuple): - expect_to_contain = expect_to_contain[index] - - result = self.get_success( - store.search_rooms([self.room_id], query, ["content.body"], 10) - ) - self.assertEquals( - result["count"], - 1 if expect_to_contain else 0, - f"expected '{query}' to match '{self.PHRASE}'" - if expect_to_contain - else f"'{query}' unexpectedly matched '{self.PHRASE}'", - ) - self.assertEquals( - len(result["results"]), - 1 if expect_to_contain else 0, - "results array length should match count", - ) - - def test_postgres_web_search_for_phrase(self): - """ - Test searching for phrases using typical web search syntax, as per postgres' websearch_to_tsquery. - This test is skipped unless the postgres instance supports websearch_to_tsquery. - """ - - store = self.hs.get_datastores().main - if not isinstance(store.database_engine, PostgresEngine): - raise SkipTest("Test only applies when postgres is used as the database") - - if store.database_engine.tsquery_func != "websearch_to_tsquery": - raise SkipTest( - "Test only applies when postgres supporting websearch_to_tsquery is used as the database" - ) - - self._check_test_cases(store, self.COMMON_CASES + self.POSTGRES_CASES, index=0) - - def test_postgres_non_web_search_for_phrase(self): - """ - Test postgres searching for phrases without using web search, which is used when websearch_to_tsquery isn't - supported by the current postgres version. - """ - - store = self.hs.get_datastores().main - if not isinstance(store.database_engine, PostgresEngine): - raise SkipTest("Test only applies when postgres is used as the database") - - # Patch supports_websearch_to_tsquery to always return False to ensure we're testing the plainto_tsquery path. - with patch( - "synapse.storage.engines.postgres.PostgresEngine.tsquery_func", - new_callable=PropertyMock, - ) as supports_websearch_to_tsquery: - supports_websearch_to_tsquery.return_value = "plainto_tsquery" - self._check_test_cases( - store, self.COMMON_CASES + self.POSTGRES_CASES, index=1 - ) - - def test_sqlite_search(self): - """ - Test sqlite searching for phrases. - """ - store = self.hs.get_datastores().main - if not isinstance(store.database_engine, Sqlite3Engine): - raise SkipTest("Test only applies when sqlite is used as the database") - - self._check_test_cases(store, self.COMMON_CASES, index=0) From 24409c7c158c108fa31433dd20ee33d423f5cb04 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 14 Feb 2018 13:47:14 +0000 Subject: [PATCH 174/178] Disable auto search for prefixes in event search --- synapse/storage/databases/main/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index 1b79acf955..ddcda6f1a7 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -771,7 +771,7 @@ def _parse_query(database_engine: BaseDatabaseEngine, search_term: str) -> str: results = re.findall(r"([\w\-]+)", search_term, re.UNICODE) if isinstance(database_engine, PostgresEngine): - return " & ".join(result + ":*" for result in results) + return " & ".join(result for result in results) elif isinstance(database_engine, Sqlite3Engine): return " & ".join(result + "*" for result in results) else: From f8f01b869e20df9628b90e0343b0c2ea9c660933 Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Fri, 4 Nov 2022 11:38:23 +0000 Subject: [PATCH 175/178] Revert "Disable auto search for prefixes in event search" This reverts commit 24409c7c158c108fa31433dd20ee33d423f5cb04. --- synapse/storage/databases/main/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index ddcda6f1a7..1b79acf955 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -771,7 +771,7 @@ def _parse_query(database_engine: BaseDatabaseEngine, search_term: str) -> str: results = re.findall(r"([\w\-]+)", search_term, re.UNICODE) if isinstance(database_engine, PostgresEngine): - return " & ".join(result for result in results) + return " & ".join(result + ":*" for result in results) elif isinstance(database_engine, Sqlite3Engine): return " & ".join(result + "*" for result in results) else: From 3b44a7c9d1beebc1dff41ab4dfe63ba7d107db45 Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Fri, 4 Nov 2022 11:38:31 +0000 Subject: [PATCH 176/178] Revert "Revert "Unified search query syntax using the full-text search capabilities of the underlying DB. (#11635)"" This reverts commit 7e0dd52782806592df61a1a517660b694515dc27. --- synapse/storage/databases/main/search.py | 197 +++++++++++++--- synapse/storage/engines/postgres.py | 16 ++ .../73/10_update_sqlite_fts4_tokenizer.py | 62 +++++ tests/storage/test_room_search.py | 213 ++++++++++++++++++ 4 files changed, 453 insertions(+), 35 deletions(-) create mode 100644 synapse/storage/schema/main/delta/73/10_update_sqlite_fts4_tokenizer.py diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index 1b79acf955..a89fc54c2c 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -11,10 +11,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import enum import logging import re -from typing import TYPE_CHECKING, Any, Collection, Iterable, List, Optional, Set, Tuple +from collections import deque +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + Any, + Collection, + Iterable, + List, + Optional, + Set, + Tuple, + Union, +) import attr @@ -27,7 +39,7 @@ LoggingTransaction, ) from synapse.storage.databases.main.events_worker import EventRedactBehaviour -from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine +from synapse.storage.engines import PostgresEngine, Sqlite3Engine from synapse.types import JsonDict if TYPE_CHECKING: @@ -421,8 +433,6 @@ async def search_msgs( """ clauses = [] - search_query = _parse_query(self.database_engine, search_term) - args: List[Any] = [] # Make sure we don't explode because the person is in too many rooms. @@ -444,20 +454,24 @@ async def search_msgs( count_clauses = clauses if isinstance(self.database_engine, PostgresEngine): + search_query = search_term + tsquery_func = self.database_engine.tsquery_func sql = ( - "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) AS rank," + f"SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) AS rank," " room_id, event_id" " FROM event_search" - " WHERE vector @@ to_tsquery('english', ?)" + f" WHERE vector @@ {tsquery_func}('english', ?)" ) args = [search_query, search_query] + args count_sql = ( "SELECT room_id, count(*) as count FROM event_search" - " WHERE vector @@ to_tsquery('english', ?)" + f" WHERE vector @@ {tsquery_func}('english', ?)" ) count_args = [search_query] + count_args elif isinstance(self.database_engine, Sqlite3Engine): + search_query = _parse_query_for_sqlite(search_term) + sql = ( "SELECT rank(matchinfo(event_search)) as rank, room_id, event_id" " FROM event_search" @@ -469,7 +483,7 @@ async def search_msgs( "SELECT room_id, count(*) as count FROM event_search" " WHERE value MATCH ?" ) - count_args = [search_term] + count_args + count_args = [search_query] + count_args else: # This should be unreachable. raise Exception("Unrecognized database engine") @@ -501,7 +515,9 @@ async def search_msgs( highlights = None if isinstance(self.database_engine, PostgresEngine): - highlights = await self._find_highlights_in_postgres(search_query, events) + highlights = await self._find_highlights_in_postgres( + search_query, events, tsquery_func + ) count_sql += " GROUP BY room_id" @@ -510,7 +526,6 @@ async def search_msgs( ) count = sum(row["count"] for row in count_results if row["room_id"] in room_ids) - return { "results": [ {"event": event_map[r["event_id"]], "rank": r["rank"]} @@ -542,9 +557,6 @@ async def search_rooms( Each match as a dictionary. """ clauses = [] - - search_query = _parse_query(self.database_engine, search_term) - args: List[Any] = [] # Make sure we don't explode because the person is in too many rooms. @@ -582,20 +594,23 @@ async def search_rooms( args.extend([origin_server_ts, origin_server_ts, stream]) if isinstance(self.database_engine, PostgresEngine): + search_query = search_term + tsquery_func = self.database_engine.tsquery_func sql = ( - "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) as rank," + f"SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) as rank," " origin_server_ts, stream_ordering, room_id, event_id" " FROM event_search" - " WHERE vector @@ to_tsquery('english', ?) AND " + f" WHERE vector @@ {tsquery_func}('english', ?) AND " ) args = [search_query, search_query] + args count_sql = ( "SELECT room_id, count(*) as count FROM event_search" - " WHERE vector @@ to_tsquery('english', ?) AND " + f" WHERE vector @@ {tsquery_func}('english', ?) AND " ) count_args = [search_query] + count_args elif isinstance(self.database_engine, Sqlite3Engine): + # We use CROSS JOIN here to ensure we use the right indexes. # https://sqlite.org/optoverview.html#crossjoin # @@ -614,13 +629,14 @@ async def search_rooms( " CROSS JOIN events USING (event_id)" " WHERE " ) + search_query = _parse_query_for_sqlite(search_term) args = [search_query] + args count_sql = ( "SELECT room_id, count(*) as count FROM event_search" " WHERE value MATCH ? AND " ) - count_args = [search_term] + count_args + count_args = [search_query] + count_args else: # This should be unreachable. raise Exception("Unrecognized database engine") @@ -660,7 +676,9 @@ async def search_rooms( highlights = None if isinstance(self.database_engine, PostgresEngine): - highlights = await self._find_highlights_in_postgres(search_query, events) + highlights = await self._find_highlights_in_postgres( + search_query, events, tsquery_func + ) count_sql += " GROUP BY room_id" @@ -686,7 +704,7 @@ async def search_rooms( } async def _find_highlights_in_postgres( - self, search_query: str, events: List[EventBase] + self, search_query: str, events: List[EventBase], tsquery_func: str ) -> Set[str]: """Given a list of events and a search term, return a list of words that match from the content of the event. @@ -697,6 +715,7 @@ async def _find_highlights_in_postgres( Args: search_query events: A list of events + tsquery_func: The tsquery_* function to use when making queries Returns: A set of strings. @@ -729,7 +748,7 @@ def f(txn: LoggingTransaction) -> Set[str]: while stop_sel in value: stop_sel += ">" - query = "SELECT ts_headline(?, to_tsquery('english', ?), %s)" % ( + query = f"SELECT ts_headline(?, {tsquery_func}('english', ?), %s)" % ( _to_postgres_options( { "StartSel": start_sel, @@ -760,20 +779,128 @@ def _to_postgres_options(options_dict: JsonDict) -> str: return "'%s'" % (",".join("%s=%s" % (k, v) for k, v in options_dict.items()),) -def _parse_query(database_engine: BaseDatabaseEngine, search_term: str) -> str: - """Takes a plain unicode string from the user and converts it into a form - that can be passed to database. - We use this so that we can add prefix matching, which isn't something - that is supported by default. +@dataclass +class Phrase: + phrase: List[str] + + +class SearchToken(enum.Enum): + Not = enum.auto() + Or = enum.auto() + And = enum.auto() + + +Token = Union[str, Phrase, SearchToken] +TokenList = List[Token] + + +def _is_stop_word(word: str) -> bool: + # TODO Pull these out of the dictionary: + # https://github.com/postgres/postgres/blob/master/src/backend/snowball/stopwords/english.stop + return word in {"the", "a", "you", "me", "and", "but"} + + +def _tokenize_query(query: str) -> TokenList: + """ + Convert the user-supplied `query` into a TokenList, which can be translated into + some DB-specific syntax. + + The following constructs are supported: + + - phrase queries using "double quotes" + - case-insensitive `or` and `and` operators + - negation of a keyword via unary `-` + - unary hyphen to denote NOT e.g. 'include -exclude' + + The following differs from websearch_to_tsquery: + + - Stop words are not removed. + - Unclosed phrases are treated differently. + + """ + tokens: TokenList = [] + + # Find phrases. + in_phrase = False + parts = deque(query.split('"')) + for i, part in enumerate(parts): + # The contents inside double quotes is treated as a phrase, a trailing + # double quote is not implied. + in_phrase = bool(i % 2) and i != (len(parts) - 1) + + # Pull out the individual words, discarding any non-word characters. + words = deque(re.findall(r"([\w\-]+)", part, re.UNICODE)) + + # Phrases have simplified handling of words. + if in_phrase: + # Skip stop words. + phrase = [word for word in words if not _is_stop_word(word)] + + # Consecutive words are implicitly ANDed together. + if tokens and tokens[-1] not in (SearchToken.Not, SearchToken.Or): + tokens.append(SearchToken.And) + + # Add the phrase. + tokens.append(Phrase(phrase)) + continue + + # Otherwise, not in a phrase. + while words: + word = words.popleft() + + if word.startswith("-"): + tokens.append(SearchToken.Not) + + # If there's more word, put it back to be processed again. + word = word[1:] + if word: + words.appendleft(word) + elif word.lower() == "or": + tokens.append(SearchToken.Or) + else: + # Skip stop words. + if _is_stop_word(word): + continue + + # Consecutive words are implicitly ANDed together. + if tokens and tokens[-1] not in (SearchToken.Not, SearchToken.Or): + tokens.append(SearchToken.And) + + # Add the search term. + tokens.append(word) + + return tokens + + +def _tokens_to_sqlite_match_query(tokens: TokenList) -> str: + """ + Convert the list of tokens to a string suitable for passing to sqlite's MATCH. + Assume sqlite was compiled with enhanced query syntax. + + Ref: https://www.sqlite.org/fts3.html#full_text_index_queries """ + match_query = [] + for token in tokens: + if isinstance(token, str): + match_query.append(token) + elif isinstance(token, Phrase): + match_query.append('"' + " ".join(token.phrase) + '"') + elif token == SearchToken.Not: + # TODO: SQLite treats NOT as a *binary* operator. Hopefully a search + # term has already been added before this. + match_query.append(" NOT ") + elif token == SearchToken.Or: + match_query.append(" OR ") + elif token == SearchToken.And: + match_query.append(" AND ") + else: + raise ValueError(f"unknown token {token}") + + return "".join(match_query) - # Pull out the individual words, discarding any non-word characters. - results = re.findall(r"([\w\-]+)", search_term, re.UNICODE) - if isinstance(database_engine, PostgresEngine): - return " & ".join(result + ":*" for result in results) - elif isinstance(database_engine, Sqlite3Engine): - return " & ".join(result + "*" for result in results) - else: - # This should be unreachable. - raise Exception("Unrecognized database engine") +def _parse_query_for_sqlite(search_term: str) -> str: + """Takes a plain unicode string from the user and converts it into a form + that can be passed to sqllite's matchinfo(). + """ + return _tokens_to_sqlite_match_query(_tokenize_query(search_term)) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index d8c0f64d9a..9bf74bbf59 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -170,6 +170,22 @@ def supports_returning(self) -> bool: """Do we support the `RETURNING` clause in insert/update/delete?""" return True + @property + def tsquery_func(self) -> str: + """ + Selects a tsquery_* func to use. + + Ref: https://www.postgresql.org/docs/current/textsearch-controls.html + + Returns: + The function name. + """ + # Postgres 11 added support for websearch_to_tsquery. + assert self._version is not None + if self._version >= 110000: + return "websearch_to_tsquery" + return "plainto_tsquery" + def is_deadlock(self, error: Exception) -> bool: if isinstance(error, psycopg2.DatabaseError): # https://www.postgresql.org/docs/current/static/errcodes-appendix.html diff --git a/synapse/storage/schema/main/delta/73/10_update_sqlite_fts4_tokenizer.py b/synapse/storage/schema/main/delta/73/10_update_sqlite_fts4_tokenizer.py new file mode 100644 index 0000000000..3de0a709eb --- /dev/null +++ b/synapse/storage/schema/main/delta/73/10_update_sqlite_fts4_tokenizer.py @@ -0,0 +1,62 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json + +from synapse.storage.engines import BaseDatabaseEngine, Sqlite3Engine +from synapse.storage.types import Cursor + + +def run_create(cur: Cursor, database_engine: BaseDatabaseEngine) -> None: + """ + Upgrade the event_search table to use the porter tokenizer if it isn't already + + Applies only for sqlite. + """ + if not isinstance(database_engine, Sqlite3Engine): + return + + # Rebuild the table event_search table with tokenize=porter configured. + cur.execute("DROP TABLE event_search") + cur.execute( + """ + CREATE VIRTUAL TABLE event_search + USING fts4 (tokenize=porter, event_id, room_id, sender, key, value ) + """ + ) + + # Re-run the background job to re-populate the event_search table. + cur.execute("SELECT MIN(stream_ordering) FROM events") + row = cur.fetchone() + min_stream_id = row[0] + + # If there are not any events, nothing to do. + if min_stream_id is None: + return + + cur.execute("SELECT MAX(stream_ordering) FROM events") + row = cur.fetchone() + max_stream_id = row[0] + + progress = { + "target_min_stream_id_inclusive": min_stream_id, + "max_stream_id_exclusive": max_stream_id + 1, + } + progress_json = json.dumps(progress) + + sql = """ + INSERT into background_updates (ordering, update_name, progress_json) + VALUES (?, ?, ?) + """ + + cur.execute(sql, (7310, "event_search", progress_json)) diff --git a/tests/storage/test_room_search.py b/tests/storage/test_room_search.py index e747c6b50e..9ddc19900a 100644 --- a/tests/storage/test_room_search.py +++ b/tests/storage/test_room_search.py @@ -12,11 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import List, Tuple, Union +from unittest.case import SkipTest +from unittest.mock import PropertyMock, patch + +from twisted.test.proto_helpers import MemoryReactor + import synapse.rest.admin from synapse.api.constants import EventTypes from synapse.api.errors import StoreError from synapse.rest.client import login, room +from synapse.server import HomeServer +from synapse.storage.databases.main import DataStore +from synapse.storage.databases.main.search import Phrase, SearchToken, _tokenize_query from synapse.storage.engines import PostgresEngine +from synapse.storage.engines.sqlite import Sqlite3Engine +from synapse.util import Clock from tests.unittest import HomeserverTestCase, skip_unless from tests.utils import USE_POSTGRES_FOR_TESTS @@ -187,3 +198,205 @@ def test_sqlite_non_string_deletion_background_update(self): ), ) self.assertCountEqual(values, ["hi", "2"]) + + +class MessageSearchTest(HomeserverTestCase): + """ + Check message search. + + A powerful way to check the behaviour is to run the following in Postgres >= 11: + + # SELECT websearch_to_tsquery('english', ); + + The result can be compared to the tokenized version for SQLite and Postgres < 11. + + """ + + servlets = [ + synapse.rest.admin.register_servlets_for_client_rest_resource, + login.register_servlets, + room.register_servlets, + ] + + PHRASE = "the quick brown fox jumps over the lazy dog" + + # Each entry is a search query, followed by either a boolean of whether it is + # in the phrase OR a tuple of booleans: whether it matches using websearch + # and using plain search. + COMMON_CASES: List[Tuple[str, Union[bool, Tuple[bool, bool]]]] = [ + ("nope", False), + ("brown", True), + ("quick brown", True), + ("brown quick", True), + ("quick \t brown", True), + ("jump", True), + ("brown nope", False), + ('"brown quick"', (False, True)), + ('"jumps over"', True), + ('"quick fox"', (False, True)), + ("nope OR doublenope", False), + ("furphy OR fox", (True, False)), + ("fox -nope", (True, False)), + ("fox -brown", (False, True)), + ('"fox" quick', True), + ('"fox quick', True), + ('"quick brown', True), + ('" quick "', True), + ('" nope"', False), + ] + # TODO Test non-ASCII cases. + + # Case that fail on SQLite. + POSTGRES_CASES: List[Tuple[str, Union[bool, Tuple[bool, bool]]]] = [ + # SQLite treats NOT as a binary operator. + ("- fox", (False, True)), + ("- nope", (True, False)), + ('"-fox quick', (False, True)), + # PostgreSQL skips stop words. + ('"the quick brown"', True), + ('"over lazy"', True), + ] + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Register a user and create a room, create some messages + self.register_user("alice", "password") + self.access_token = self.login("alice", "password") + self.room_id = self.helper.create_room_as("alice", tok=self.access_token) + + # Send the phrase as a message and check it was created + response = self.helper.send(self.room_id, self.PHRASE, tok=self.access_token) + self.assertIn("event_id", response) + + def test_tokenize_query(self) -> None: + """Test the custom logic to tokenize a user's query.""" + cases = ( + ("brown", ["brown"]), + ("quick brown", ["quick", SearchToken.And, "brown"]), + ("quick \t brown", ["quick", SearchToken.And, "brown"]), + ('"brown quick"', [Phrase(["brown", "quick"])]), + ("furphy OR fox", ["furphy", SearchToken.Or, "fox"]), + ("fox -brown", ["fox", SearchToken.Not, "brown"]), + ("- fox", [SearchToken.Not, "fox"]), + ('"fox" quick', [Phrase(["fox"]), SearchToken.And, "quick"]), + # No trailing double quoe. + ('"fox quick', ["fox", SearchToken.And, "quick"]), + ('"-fox quick', [SearchToken.Not, "fox", SearchToken.And, "quick"]), + ('" quick "', [Phrase(["quick"])]), + ( + 'q"uick brow"n', + [ + "q", + SearchToken.And, + Phrase(["uick", "brow"]), + SearchToken.And, + "n", + ], + ), + ( + '-"quick brown"', + [SearchToken.Not, Phrase(["quick", "brown"])], + ), + ) + + for query, expected in cases: + tokenized = _tokenize_query(query) + self.assertEqual( + tokenized, expected, f"{tokenized} != {expected} for {query}" + ) + + def _check_test_cases( + self, + store: DataStore, + cases: List[Tuple[str, Union[bool, Tuple[bool, bool]]]], + index=0, + ) -> None: + # Run all the test cases versus search_msgs + for query, expect_to_contain in cases: + if isinstance(expect_to_contain, tuple): + expect_to_contain = expect_to_contain[index] + + result = self.get_success( + store.search_msgs([self.room_id], query, ["content.body"]) + ) + self.assertEquals( + result["count"], + 1 if expect_to_contain else 0, + f"expected '{query}' to match '{self.PHRASE}'" + if expect_to_contain + else f"'{query}' unexpectedly matched '{self.PHRASE}'", + ) + self.assertEquals( + len(result["results"]), + 1 if expect_to_contain else 0, + "results array length should match count", + ) + + # Run them again versus search_rooms + for query, expect_to_contain in cases: + if isinstance(expect_to_contain, tuple): + expect_to_contain = expect_to_contain[index] + + result = self.get_success( + store.search_rooms([self.room_id], query, ["content.body"], 10) + ) + self.assertEquals( + result["count"], + 1 if expect_to_contain else 0, + f"expected '{query}' to match '{self.PHRASE}'" + if expect_to_contain + else f"'{query}' unexpectedly matched '{self.PHRASE}'", + ) + self.assertEquals( + len(result["results"]), + 1 if expect_to_contain else 0, + "results array length should match count", + ) + + def test_postgres_web_search_for_phrase(self): + """ + Test searching for phrases using typical web search syntax, as per postgres' websearch_to_tsquery. + This test is skipped unless the postgres instance supports websearch_to_tsquery. + """ + + store = self.hs.get_datastores().main + if not isinstance(store.database_engine, PostgresEngine): + raise SkipTest("Test only applies when postgres is used as the database") + + if store.database_engine.tsquery_func != "websearch_to_tsquery": + raise SkipTest( + "Test only applies when postgres supporting websearch_to_tsquery is used as the database" + ) + + self._check_test_cases(store, self.COMMON_CASES + self.POSTGRES_CASES, index=0) + + def test_postgres_non_web_search_for_phrase(self): + """ + Test postgres searching for phrases without using web search, which is used when websearch_to_tsquery isn't + supported by the current postgres version. + """ + + store = self.hs.get_datastores().main + if not isinstance(store.database_engine, PostgresEngine): + raise SkipTest("Test only applies when postgres is used as the database") + + # Patch supports_websearch_to_tsquery to always return False to ensure we're testing the plainto_tsquery path. + with patch( + "synapse.storage.engines.postgres.PostgresEngine.tsquery_func", + new_callable=PropertyMock, + ) as supports_websearch_to_tsquery: + supports_websearch_to_tsquery.return_value = "plainto_tsquery" + self._check_test_cases( + store, self.COMMON_CASES + self.POSTGRES_CASES, index=1 + ) + + def test_sqlite_search(self): + """ + Test sqlite searching for phrases. + """ + store = self.hs.get_datastores().main + if not isinstance(store.database_engine, Sqlite3Engine): + raise SkipTest("Test only applies when sqlite is used as the database") + + self._check_test_cases(store, self.COMMON_CASES, index=0) From e2008e4ef5235fafd4dea72c73472faa379c1e3f Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Fri, 4 Nov 2022 11:38:39 +0000 Subject: [PATCH 177/178] Revert "Revert "Fix tests for change in PostgreSQL 14 behavior change. (#14310)"" This reverts commit 7f77f1386c94c753755b79d7d7025e958b228282. --- synapse/storage/databases/main/search.py | 5 ++--- tests/storage/test_room_search.py | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index a89fc54c2c..594b935614 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -824,9 +824,8 @@ def _tokenize_query(query: str) -> TokenList: in_phrase = False parts = deque(query.split('"')) for i, part in enumerate(parts): - # The contents inside double quotes is treated as a phrase, a trailing - # double quote is not implied. - in_phrase = bool(i % 2) and i != (len(parts) - 1) + # The contents inside double quotes is treated as a phrase. + in_phrase = bool(i % 2) # Pull out the individual words, discarding any non-word characters. words = deque(re.findall(r"([\w\-]+)", part, re.UNICODE)) diff --git a/tests/storage/test_room_search.py b/tests/storage/test_room_search.py index 9ddc19900a..868b5bee84 100644 --- a/tests/storage/test_room_search.py +++ b/tests/storage/test_room_search.py @@ -239,7 +239,6 @@ class MessageSearchTest(HomeserverTestCase): ("fox -nope", (True, False)), ("fox -brown", (False, True)), ('"fox" quick', True), - ('"fox quick', True), ('"quick brown', True), ('" quick "', True), ('" nope"', False), @@ -269,6 +268,15 @@ def prepare( response = self.helper.send(self.room_id, self.PHRASE, tok=self.access_token) self.assertIn("event_id", response) + # The behaviour of a missing trailing double quote changed in PostgreSQL 14 + # from ignoring the initial double quote to treating it as a phrase. + main_store = homeserver.get_datastores().main + found = False + if isinstance(main_store.database_engine, PostgresEngine): + assert main_store.database_engine._version is not None + found = main_store.database_engine._version < 140000 + self.COMMON_CASES.append(('"fox quick', (found, True))) + def test_tokenize_query(self) -> None: """Test the custom logic to tokenize a user's query.""" cases = ( @@ -280,9 +288,9 @@ def test_tokenize_query(self) -> None: ("fox -brown", ["fox", SearchToken.Not, "brown"]), ("- fox", [SearchToken.Not, "fox"]), ('"fox" quick', [Phrase(["fox"]), SearchToken.And, "quick"]), - # No trailing double quoe. - ('"fox quick', ["fox", SearchToken.And, "quick"]), - ('"-fox quick', [SearchToken.Not, "fox", SearchToken.And, "quick"]), + # No trailing double quote. + ('"fox quick', [Phrase(["fox", "quick"])]), + ('"-fox quick', [Phrase(["-fox", "quick"])]), ('" quick "', [Phrase(["quick"])]), ( 'q"uick brow"n', From 02a25ce3f934451b34105a4fb7a6366e2f563b99 Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Fri, 4 Nov 2022 11:38:46 +0000 Subject: [PATCH 178/178] Revert "Revert "Switch search SQL to triple-quote strings. (#14311)"" This reverts commit 220af1df54c3b4e5f1d7ec8fb62b375c99a2bbe2. --- synapse/storage/databases/main/search.py | 188 ++++++++++++----------- 1 file changed, 99 insertions(+), 89 deletions(-) diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index 594b935614..e9588d1755 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -80,11 +80,11 @@ def store_search_entries_txn( if not self.hs.config.server.enable_search: return if isinstance(self.database_engine, PostgresEngine): - sql = ( - "INSERT INTO event_search" - " (event_id, room_id, key, vector, stream_ordering, origin_server_ts)" - " VALUES (?,?,?,to_tsvector('english', ?),?,?)" - ) + sql = """ + INSERT INTO event_search + (event_id, room_id, key, vector, stream_ordering, origin_server_ts) + VALUES (?,?,?,to_tsvector('english', ?),?,?) + """ args1 = ( ( @@ -101,20 +101,20 @@ def store_search_entries_txn( txn.execute_batch(sql, args1) elif isinstance(self.database_engine, Sqlite3Engine): - sql = ( - "INSERT INTO event_search (event_id, room_id, key, value)" - " VALUES (?,?,?,?)" - ) - args2 = ( - ( - entry.event_id, - entry.room_id, - entry.key, - _clean_value_for_search(entry.value), - ) - for entry in entries + self.db_pool.simple_insert_many_txn( + txn, + table="event_search", + keys=("event_id", "room_id", "key", "value"), + values=( + ( + entry.event_id, + entry.room_id, + entry.key, + _clean_value_for_search(entry.value), + ) + for entry in entries + ), ) - txn.execute_batch(sql, args2) else: # This should be unreachable. @@ -162,15 +162,17 @@ async def _background_reindex_search( TYPES = ["m.room.name", "m.room.message", "m.room.topic"] def reindex_search_txn(txn: LoggingTransaction) -> int: - sql = ( - "SELECT stream_ordering, event_id, room_id, type, json, " - " origin_server_ts FROM events" - " JOIN event_json USING (room_id, event_id)" - " WHERE ? <= stream_ordering AND stream_ordering < ?" - " AND (%s)" - " ORDER BY stream_ordering DESC" - " LIMIT ?" - ) % (" OR ".join("type = '%s'" % (t,) for t in TYPES),) + sql = """ + SELECT stream_ordering, event_id, room_id, type, json, origin_server_ts + FROM events + JOIN event_json USING (room_id, event_id) + WHERE ? <= stream_ordering AND stream_ordering < ? + AND (%s) + ORDER BY stream_ordering DESC + LIMIT ? + """ % ( + " OR ".join("type = '%s'" % (t,) for t in TYPES), + ) txn.execute(sql, (target_min_stream_id, max_stream_id, batch_size)) @@ -284,8 +286,10 @@ def create_index(conn: LoggingDatabaseConnection) -> None: try: c.execute( - "CREATE INDEX CONCURRENTLY event_search_fts_idx" - " ON event_search USING GIN (vector)" + """ + CREATE INDEX CONCURRENTLY event_search_fts_idx + ON event_search USING GIN (vector) + """ ) except psycopg2.ProgrammingError as e: logger.warning( @@ -323,12 +327,16 @@ def create_index(conn: LoggingDatabaseConnection) -> None: # We create with NULLS FIRST so that when we search *backwards* # we get the ones with non null origin_server_ts *first* c.execute( - "CREATE INDEX CONCURRENTLY event_search_room_order ON event_search(" - "room_id, origin_server_ts NULLS FIRST, stream_ordering NULLS FIRST)" + """ + CREATE INDEX CONCURRENTLY event_search_room_order + ON event_search(room_id, origin_server_ts NULLS FIRST, stream_ordering NULLS FIRST) + """ ) c.execute( - "CREATE INDEX CONCURRENTLY event_search_order ON event_search(" - "origin_server_ts NULLS FIRST, stream_ordering NULLS FIRST)" + """ + CREATE INDEX CONCURRENTLY event_search_order + ON event_search(origin_server_ts NULLS FIRST, stream_ordering NULLS FIRST) + """ ) conn.set_session(autocommit=False) @@ -345,14 +353,14 @@ def create_index(conn: LoggingDatabaseConnection) -> None: ) def reindex_search_txn(txn: LoggingTransaction) -> Tuple[int, bool]: - sql = ( - "UPDATE event_search AS es SET stream_ordering = e.stream_ordering," - " origin_server_ts = e.origin_server_ts" - " FROM events AS e" - " WHERE e.event_id = es.event_id" - " AND ? <= e.stream_ordering AND e.stream_ordering < ?" - " RETURNING es.stream_ordering" - ) + sql = """ + UPDATE event_search AS es + SET stream_ordering = e.stream_ordering, origin_server_ts = e.origin_server_ts + FROM events AS e + WHERE e.event_id = es.event_id + AND ? <= e.stream_ordering AND e.stream_ordering < ? + RETURNING es.stream_ordering + """ min_stream_id = max_stream_id - batch_size txn.execute(sql, (min_stream_id, max_stream_id)) @@ -456,33 +464,33 @@ async def search_msgs( if isinstance(self.database_engine, PostgresEngine): search_query = search_term tsquery_func = self.database_engine.tsquery_func - sql = ( - f"SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) AS rank," - " room_id, event_id" - " FROM event_search" - f" WHERE vector @@ {tsquery_func}('english', ?)" - ) + sql = f""" + SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) AS rank, + room_id, event_id + FROM event_search + WHERE vector @@ {tsquery_func}('english', ?) + """ args = [search_query, search_query] + args - count_sql = ( - "SELECT room_id, count(*) as count FROM event_search" - f" WHERE vector @@ {tsquery_func}('english', ?)" - ) + count_sql = f""" + SELECT room_id, count(*) as count FROM event_search + WHERE vector @@ {tsquery_func}('english', ?) + """ count_args = [search_query] + count_args elif isinstance(self.database_engine, Sqlite3Engine): search_query = _parse_query_for_sqlite(search_term) - sql = ( - "SELECT rank(matchinfo(event_search)) as rank, room_id, event_id" - " FROM event_search" - " WHERE value MATCH ?" - ) + sql = """ + SELECT rank(matchinfo(event_search)) as rank, room_id, event_id + FROM event_search + WHERE value MATCH ? + """ args = [search_query] + args - count_sql = ( - "SELECT room_id, count(*) as count FROM event_search" - " WHERE value MATCH ?" - ) + count_sql = """ + SELECT room_id, count(*) as count FROM event_search + WHERE value MATCH ? + """ count_args = [search_query] + count_args else: # This should be unreachable. @@ -588,26 +596,27 @@ async def search_rooms( raise SynapseError(400, "Invalid pagination token") clauses.append( - "(origin_server_ts < ?" - " OR (origin_server_ts = ? AND stream_ordering < ?))" + """ + (origin_server_ts < ? OR (origin_server_ts = ? AND stream_ordering < ?)) + """ ) args.extend([origin_server_ts, origin_server_ts, stream]) if isinstance(self.database_engine, PostgresEngine): search_query = search_term tsquery_func = self.database_engine.tsquery_func - sql = ( - f"SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) as rank," - " origin_server_ts, stream_ordering, room_id, event_id" - " FROM event_search" - f" WHERE vector @@ {tsquery_func}('english', ?) AND " - ) + sql = f""" + SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) as rank, + origin_server_ts, stream_ordering, room_id, event_id + FROM event_search + WHERE vector @@ {tsquery_func}('english', ?) AND + """ args = [search_query, search_query] + args - count_sql = ( - "SELECT room_id, count(*) as count FROM event_search" - f" WHERE vector @@ {tsquery_func}('english', ?) AND " - ) + count_sql = f""" + SELECT room_id, count(*) as count FROM event_search + WHERE vector @@ {tsquery_func}('english', ?) AND + """ count_args = [search_query] + count_args elif isinstance(self.database_engine, Sqlite3Engine): @@ -619,23 +628,24 @@ async def search_rooms( # in the events table to get the topological ordering. We need # to use the indexes in this order because sqlite refuses to # MATCH unless it uses the full text search index - sql = ( - "SELECT rank(matchinfo) as rank, room_id, event_id," - " origin_server_ts, stream_ordering" - " FROM (SELECT key, event_id, matchinfo(event_search) as matchinfo" - " FROM event_search" - " WHERE value MATCH ?" - " )" - " CROSS JOIN events USING (event_id)" - " WHERE " + sql = """ + SELECT + rank(matchinfo) as rank, room_id, event_id, origin_server_ts, stream_ordering + FROM ( + SELECT key, event_id, matchinfo(event_search) as matchinfo + FROM event_search + WHERE value MATCH ? ) + CROSS JOIN events USING (event_id) + WHERE + """ search_query = _parse_query_for_sqlite(search_term) args = [search_query] + args - count_sql = ( - "SELECT room_id, count(*) as count FROM event_search" - " WHERE value MATCH ? AND " - ) + count_sql = """ + SELECT room_id, count(*) as count FROM event_search + WHERE value MATCH ? AND + """ count_args = [search_query] + count_args else: # This should be unreachable. @@ -647,10 +657,10 @@ async def search_rooms( # We add an arbitrary limit here to ensure we don't try to pull the # entire table from the database. if isinstance(self.database_engine, PostgresEngine): - sql += ( - " ORDER BY origin_server_ts DESC NULLS LAST," - " stream_ordering DESC NULLS LAST LIMIT ?" - ) + sql += """ + ORDER BY origin_server_ts DESC NULLS LAST, stream_ordering DESC NULLS LAST + LIMIT ? + """ elif isinstance(self.database_engine, Sqlite3Engine): sql += " ORDER BY origin_server_ts DESC, stream_ordering DESC LIMIT ?" else: