diff --git a/.taskcluster.yml b/.taskcluster.yml index 4c37614c2..c8034ae7c 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -149,7 +149,7 @@ tasks: - sh - -lxce - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && - cd /src/report && ./ci-test.sh" + cd /src/report && pip install -r requirements.txt && ./ci-test.sh" metadata: name: "Code Coverage Report checks: unit tests" description: Check python code with unittest diff --git a/report/firefox_code_coverage/__init__.py b/report/firefox_code_coverage/__init__.py index 40a96afc6..212e90bdb 100644 --- a/report/firefox_code_coverage/__init__.py +++ b/report/firefox_code_coverage/__init__.py @@ -1 +1,8 @@ # -*- coding: utf-8 -*- + +from taskcluster.helper import TaskclusterConfig + +taskcluster = TaskclusterConfig("https://firefox-ci-tc.services.mozilla.com") + +# Force root url to avoid proxy as grcov is not available on Community instance +taskcluster.options = {"rootUrl": taskcluster.default_url} diff --git a/report/firefox_code_coverage/codecoverage.py b/report/firefox_code_coverage/codecoverage.py index 97e289e08..123c0ee8c 100644 --- a/report/firefox_code_coverage/codecoverage.py +++ b/report/firefox_code_coverage/codecoverage.py @@ -12,13 +12,9 @@ import time import warnings -try: - from urllib.parse import urlencode - from urllib.request import Request, urlopen, urlretrieve -except ImportError: - from urllib import urlencode, urlretrieve - from urllib2 import Request, urlopen +import requests +from firefox_code_coverage import taskcluster FINISHED_STATUSES = ["completed", "failed", "exception"] ALL_STATUSES = FINISHED_STATUSES + ["unscheduled", "pending", "running"] @@ -28,91 +24,87 @@ GRCOV_ARTIFACT = "public/build/grcov.tar.xz" -def get_json(url, params=None, headers={}): - if params is not None: - url += "?" + urlencode(params) - - request = Request(url, headers=headers) - r = urlopen(request).read().decode("utf-8") - - return json.loads(r) - - def is_taskcluster_loaner(): return "TASKCLUSTER_INTERACTIVE" in os.environ def get_task(branch, revision): - task = get_json( - f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.{branch}.revision.{revision}.firefox.decision" - ) + index = taskcluster.get_service("index") + task = index.findTask(f"gecko.v2.{branch}.revision.{revision}.firefox.decision") return task["taskId"] def get_last_task(): - revision = get_json( + resp = requests.get( "https://api.coverage.moz.tools/v2/latest?repository=mozilla-central" - )[0]["revision"] + ) + resp.raise_for_status() + data = resp.json() + revision = data[0]["revision"] return get_task("mozilla-central", revision) def get_task_details(task_id): - task_details = get_json( - "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/" + task_id - ) - return task_details + queue = taskcluster.get_service("queue") + return queue.task(task_id) def get_task_artifacts(task_id): - artifacts = get_json( - "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/" - + task_id - + "/artifacts" - ) - return artifacts["artifacts"] + queue = taskcluster.get_service("queue") + response = queue.listLatestArtifacts(task_id) + return response["artifacts"] def get_tasks_in_group(group_id): - reply = get_json( - "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task-group/" - + group_id - + "/list", - {"limit": "200"}, - ) - tasks = reply["tasks"] - while "continuationToken" in reply: - reply = get_json( - "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task-group/" - + group_id - + "/list", - {"limit": "200", "continuationToken": reply["continuationToken"]}, - ) - tasks += reply["tasks"] + tasks = [] + + def _save_tasks(response): + tasks.extend(response["tasks"]) + + queue = taskcluster.get_service("queue") + queue.listTaskGroup(group_id, paginationHandler=_save_tasks) + return tasks +def download_binary(url, path, retries=5): + """Download a binary file from an url""" + for i in range(1, retries + 1): + try: + artifact = requests.get(url, stream=True) + artifact.raise_for_status() + + with open(path, "wb") as f: + for chunk in artifact.iter_content(chunk_size=8192): + f.write(chunk) + break + except: # noqa: E722 + try: + os.remove(path) + except OSError: + pass + + if i == retries: + raise Exception( + "Download failed after {} retries - {}".format(retries, url) + ) + + time.sleep(7 * i) + + def download_artifact(task_id, artifact, artifacts_path): fname = os.path.join( artifacts_path, task_id + "_" + os.path.basename(artifact["name"]) ) + + # As recommended by Taskcluster doc, use requests to download + # from the artifact public url instead of relying on the client method + queue = taskcluster.get_service("queue") + url = queue.buildUrl("getLatestArtifact", task_id, artifact["name"]) + if not os.path.exists(fname): - while True: - try: - urlretrieve( - "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/" - + task_id - + "/artifacts/" - + artifact["name"], - fname, - ) - break - except: # noqa: E722 - try: - os.remove(fname) - except OSError: - pass - - time.sleep(7) + download_binary(url, fname) + return fname @@ -145,11 +137,8 @@ def get_platform(task_name): def get_task_status(task_id): - status = get_json( - "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{}/status".format( - task_id - ) - ) + queue = taskcluster.get_service("queue") + status = queue.status(task_id) return status["status"]["state"] @@ -316,10 +305,9 @@ def download_grcov(): dest = tempfile.mkdtemp(suffix="grcov") archive = os.path.join(dest, "grcov.tar.xz") - url = "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/{index}/artifacts/{artifact}".format( - index=GRCOV_INDEX, artifact=GRCOV_ARTIFACT - ) - urlretrieve(url, archive) + index = taskcluster.get_service("index") + url = index.buildUrl("findArtifactFromTask", GRCOV_INDEX, GRCOV_ARTIFACT) + download_binary(url, archive) # Extract archive in temp tar = tarfile.open(archive, "r:xz") diff --git a/report/requirements.txt b/report/requirements.txt new file mode 100644 index 000000000..28744c326 --- /dev/null +++ b/report/requirements.txt @@ -0,0 +1 @@ +taskcluster==24.2.0