From 6c777bdc75a2fe260d7bff0abd69d6cd5bd32014 Mon Sep 17 00:00:00 2001 From: nitish naineni Date: Tue, 4 Feb 2020 03:04:26 +0530 Subject: [PATCH 1/2] using tenacity in download_binary function --- report/firefox_code_coverage/codecoverage.py | 40 ++++++++++---------- report/requirements.txt | 1 + 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/report/firefox_code_coverage/codecoverage.py b/report/firefox_code_coverage/codecoverage.py index 4cc6d9498..0cd106e7a 100644 --- a/report/firefox_code_coverage/codecoverage.py +++ b/report/firefox_code_coverage/codecoverage.py @@ -11,7 +11,7 @@ import tempfile import time import warnings - +import tenacity import requests from firefox_code_coverage import taskcluster @@ -67,29 +67,27 @@ def _save_tasks(response): return tasks -def download_binary(url, path, retries=5): +@tenacity.retry(stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_incrementing(start = 7, increment = 7), + reraise=True) +def download_binary(url, path): """Download a binary file from an url""" - for i in range(1, retries + 1): + + try: + artifact = requests.get(url, stream=True) + artifact.raise_for_status() + + with open(path, "wb") as f: + for chunk in artifact.iter_content(chunk_size=8192): + f.write(chunk) + + except Exception: try: - artifact = requests.get(url, stream=True) - artifact.raise_for_status() - - with open(path, "wb") as f: - for chunk in artifact.iter_content(chunk_size=8192): - f.write(chunk) - break - except: # noqa: E722 - try: - os.remove(path) - except OSError: - pass - - if i == retries: - raise Exception( - "Download failed after {} retries - {}".format(retries, url) - ) + os.remove(path) + except OSError: + pass - time.sleep(7 * i) + raise def download_artifact(task_id, artifact, artifacts_path): diff --git a/report/requirements.txt b/report/requirements.txt index 28744c326..174d2f772 100644 --- a/report/requirements.txt +++ b/report/requirements.txt @@ -1 +1,2 @@ taskcluster==24.2.0 +tenacity==6.0.0 \ No newline at end of file From 0429c07f22eb5bff351b2b0edb51c3577e1bd1bb Mon Sep 17 00:00:00 2001 From: nitish naineni Date: Tue, 4 Feb 2020 03:43:34 +0530 Subject: [PATCH 2/2] using tenacity in download_binary function --- report/firefox_code_coverage/codecoverage.py | 15 +++++++++------ report/requirements.txt | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/report/firefox_code_coverage/codecoverage.py b/report/firefox_code_coverage/codecoverage.py index 0cd106e7a..e3f85ca9f 100644 --- a/report/firefox_code_coverage/codecoverage.py +++ b/report/firefox_code_coverage/codecoverage.py @@ -11,8 +11,9 @@ import tempfile import time import warnings -import tenacity + import requests +import tenacity from firefox_code_coverage import taskcluster @@ -67,12 +68,14 @@ def _save_tasks(response): return tasks -@tenacity.retry(stop=tenacity.stop_after_attempt(5), - wait=tenacity.wait_incrementing(start = 7, increment = 7), - reraise=True) +@tenacity.retry( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_incrementing(start=7, increment=7), + reraise=True, +) def download_binary(url, path): """Download a binary file from an url""" - + try: artifact = requests.get(url, stream=True) artifact.raise_for_status() @@ -80,7 +83,7 @@ def download_binary(url, path): with open(path, "wb") as f: for chunk in artifact.iter_content(chunk_size=8192): f.write(chunk) - + except Exception: try: os.remove(path) diff --git a/report/requirements.txt b/report/requirements.txt index 174d2f772..c8fd962f4 100644 --- a/report/requirements.txt +++ b/report/requirements.txt @@ -1,2 +1,2 @@ taskcluster==24.2.0 -tenacity==6.0.0 \ No newline at end of file +tenacity==6.0.0