diff --git a/backend/code_coverage_backend/gcp.py b/backend/code_coverage_backend/gcp.py index f03326322..61f03660d 100644 --- a/backend/code_coverage_backend/gcp.py +++ b/backend/code_coverage_backend/gcp.py @@ -5,7 +5,9 @@ import re import tempfile from datetime import datetime +from datetime import timedelta +import pytz import redis import structlog import zstandard as zstd @@ -368,7 +370,7 @@ def get_suites(self, repository): suites = self.redis.smembers(KEY_SUITES.format(repository=repository)) return sorted(map(lambda x: x.decode("utf-8"), suites)) - def ingest_available_reports(self, repository): + def ingest_available_reports(self, repository, until=None): """ Ingest all the available reports for a repository """ @@ -377,8 +379,13 @@ def ingest_available_reports(self, repository): REGEX_BLOB = re.compile( r"^{}/(\w+)/([\w\-]+):([\w\-]+).json.zstd$".format(repository) ) + now = datetime.utcnow().replace(tzinfo=pytz.UTC) for blob in self.bucket.list_blobs(prefix=repository): + if isinstance(until, timedelta) and (now - blob.time_created) >= until: + logger.debug(f"Skipping old blob {blob}") + continue + # Get changeset from blob name match = REGEX_BLOB.match(blob.name) if match is None: diff --git a/backend/tools/cleanup.py b/backend/tools/cleanup.py new file mode 100644 index 000000000..4901ea545 --- /dev/null +++ b/backend/tools/cleanup.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +import redis + + +def cleanup(client, prefix): + nb, memory = 0, 0 + for key in client.keys(f"{prefix}:*"): + if key.endswith(b"all:all"): + continue + + key_memory = client.memory_usage(key) + nb += 1 + memory += key_memory + print(f"Removing {key_memory}b for {key}") + + client.delete(key) + + print(f"Removed {nb} keys for {memory} bytes") + + +if __name__ == "__main__": + client = redis.from_url(os.environ["REDIS_URL"]) + cleanup(client, "overall:mozilla-central")