diff --git a/.isort.cfg b/.isort.cfg index 5f64710da..ab08dd46a 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,6 +1,6 @@ [settings] known_first_party = code_coverage_backend,code_coverage_bot,code_coverage_events,code_coverage_tools,conftest,firefox_code_coverage -known_third_party = connexion,datadog,dateutil,fakeredis,flask,flask_cors,flask_talisman,google,hglib,jsone,jsonschema,libmozdata,libmozevent,logbook,pytest,pytz,raven,redis,requests,responses,setuptools,structlog,taskcluster,werkzeug,zstandard +known_third_party = connexion,datadog,dateutil,fakeredis,flask,flask_cors,flask_talisman,google,hglib,jsone,jsonschema,libmozdata,libmozevent,logbook,pytest,pytz,raven,redis,requests,responses,setuptools,structlog,taskcluster,tenacity,werkzeug,zstandard force_single_line = True default_section=FIRSTPARTY line_length=159 diff --git a/bot/code_coverage_bot/taskcluster.py b/bot/code_coverage_bot/taskcluster.py index 8f58c247a..329b0dc71 100644 --- a/bot/code_coverage_bot/taskcluster.py +++ b/bot/code_coverage_bot/taskcluster.py @@ -7,10 +7,9 @@ import requests import structlog import taskcluster +import tenacity from taskcluster.helper import TaskclusterConfig -from code_coverage_bot.utils import retry - logger = structlog.getLogger(__name__) taskcluster_config = TaskclusterConfig("https://firefox-ci-tc.services.mozilla.com") @@ -72,6 +71,9 @@ def download_artifact(artifact_path, task_id, artifact_name): url = queue.buildUrl("getLatestArtifact", task_id, artifact_name) logger.debug("Downloading artifact", url=url) + @tenacity.retry( + reraise=True, wait=tenacity.wait_fixed(30), stop=tenacity.stop_after_attempt(5) + ) def perform_download(): r = requests.get(url, stream=True) r.raise_for_status() @@ -83,7 +85,7 @@ def perform_download(): if artifact_path.endswith(".zip") and not is_zipfile(artifact_path): raise BadZipFile("File is not a zip file") - retry(perform_download) + perform_download() def is_coverage_task(task): diff --git a/bot/code_coverage_bot/uploader.py b/bot/code_coverage_bot/uploader.py index 4b8c4a26b..929f65c4c 100644 --- a/bot/code_coverage_bot/uploader.py +++ b/bot/code_coverage_bot/uploader.py @@ -4,10 +4,10 @@ import requests import structlog +import tenacity import zstandard as zstd from code_coverage_bot.secrets import secrets -from code_coverage_bot.utils import retry from code_coverage_tools.gcp import get_bucket logger = structlog.get_logger(__name__) @@ -45,11 +45,7 @@ def gcp(repository, revision, report, platform, suite): logger.info("Uploaded {} on {}".format(path, bucket)) # Trigger ingestion on backend - retry( - lambda: gcp_ingest(repository, revision, platform, suite), - retries=10, - wait_between_retries=60, - ) + gcp_ingest(repository, revision, platform, suite) return blob @@ -66,6 +62,9 @@ def gcp_covdir_exists(repository, revision, platform, suite): return blob.exists() +@tenacity.retry( + stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_fixed(60), reraise=True +) def gcp_ingest(repository, revision, platform, suite): """ The GCP report ingestion is triggered remotely on a backend diff --git a/bot/code_coverage_bot/utils.py b/bot/code_coverage_bot/utils.py index 38c56fcef..0bafa8ff4 100644 --- a/bot/code_coverage_bot/utils.py +++ b/bot/code_coverage_bot/utils.py @@ -4,34 +4,12 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import concurrent.futures import subprocess -import time import structlog log = structlog.get_logger(__name__) -class RunException(Exception): - """ - Exception used to stop retrying - """ - - -def retry( - operation, retries=5, wait_between_retries=30, exception_to_break=RunException -): - while True: - try: - return operation() - except Exception as e: - if isinstance(e, exception_to_break): - raise - retries -= 1 - if retries == 0: - raise - time.sleep(wait_between_retries) - - def hide_secrets(text, secrets): if type(text) is bytes: encode_secret, xxx = lambda x: bytes(x, encoding="utf-8"), b"XXX" @@ -54,7 +32,7 @@ def run_check(command, **kwargs): assert isinstance(command, list) if len(command) == 0: - raise RunException("Can't run an empty command.") + raise Exception("Can't run an empty command.") _kwargs = dict( stdin=subprocess.DEVNULL, # no interactions @@ -81,7 +59,7 @@ def run_check(command, **kwargs): error=error, ) - raise RunException(f"`{command[0]}` failed with code: {proc.returncode}.") + raise Exception(f"`{command[0]}` failed with code: {proc.returncode}.") return output diff --git a/bot/requirements.txt b/bot/requirements.txt index 07b86f86d..1b47fe582 100644 --- a/bot/requirements.txt +++ b/bot/requirements.txt @@ -3,4 +3,5 @@ google-cloud-storage==1.25.0 libmozdata==0.1.64 pytoml==0.1.21 pytz==2019.3 +tenacity==6.0.0 zstandard==0.13.0 diff --git a/bot/tests/test_taskcluster.py b/bot/tests/test_taskcluster.py index 9a5bc428b..7bcf9d2fa 100644 --- a/bot/tests/test_taskcluster.py +++ b/bot/tests/test_taskcluster.py @@ -275,7 +275,7 @@ def test_download_artifact_forbidden(mocked_sleep, mock_taskcluster, tmpdir): "public/test_info/code-coverage-grcov.zip", ) - assert mocked_sleep.call_count == 4 + assert len(responses.calls) == 5 @mock.patch("time.sleep") @@ -295,4 +295,4 @@ def test_download_artifact_badzip(mocked_sleep, mock_taskcluster, tmpdir): "public/test_info/code-coverage-grcov.zip", ) - assert mocked_sleep.call_count == 4 + assert len(responses.calls) == 5