diff --git a/.isort.cfg b/.isort.cfg index ab08dd46a..310bc0c1d 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,6 +1,6 @@ [settings] known_first_party = code_coverage_backend,code_coverage_bot,code_coverage_events,code_coverage_tools,conftest,firefox_code_coverage -known_third_party = connexion,datadog,dateutil,fakeredis,flask,flask_cors,flask_talisman,google,hglib,jsone,jsonschema,libmozdata,libmozevent,logbook,pytest,pytz,raven,redis,requests,responses,setuptools,structlog,taskcluster,tenacity,werkzeug,zstandard +known_third_party = connexion,datadog,dateutil,fakeredis,flask,flask_cors,flask_talisman,google,hglib,jsone,jsonschema,libmozdata,libmozevent,logbook,magic,pytest,pytz,raven,redis,requests,responses,setuptools,structlog,taskcluster,tenacity,werkzeug,zstandard force_single_line = True default_section=FIRSTPARTY line_length=159 diff --git a/.taskcluster.yml b/.taskcluster.yml index 99588ad7f..8b9e3f2e9 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -143,13 +143,15 @@ tasks: created: {$fromNow: ''} deadline: {$fromNow: '1 hour'} payload: + features: + taskclusterProxy: true maxRunTime: 3600 image: python:3.8 command: - sh - -lxce - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && - cd /src/report && pip install -r requirements.txt && ./ci-test.sh" + cd /src/report && ./ci-test.sh" metadata: name: "Code Coverage Report checks: unit tests" description: Check python code with unittest diff --git a/report/ci-test.sh b/report/ci-test.sh index 750c87d58..b7df8c48f 100755 --- a/report/ci-test.sh +++ b/report/ci-test.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -pip install --disable-pip-version-check --no-cache-dir --quiet -r test-requirements.txt +pip install --disable-pip-version-check --no-cache-dir --quiet -r test-requirements.txt -r requirements.txt . python -m unittest discover tests python setup.py sdist bdist_wheel diff --git a/report/firefox_code_coverage/codecoverage.py b/report/firefox_code_coverage/codecoverage.py index e3f85ca9f..9a57cf402 100644 --- a/report/firefox_code_coverage/codecoverage.py +++ b/report/firefox_code_coverage/codecoverage.py @@ -3,6 +3,7 @@ import argparse import errno import json +import logging import os import shutil import subprocess @@ -11,7 +12,10 @@ import tempfile import time import warnings +from datetime import timedelta +from pathlib import Path +import magic import requests import tenacity @@ -24,6 +28,8 @@ GRCOV_INDEX = "gecko.cache.level-3.toolchains.v3.linux64-grcov.latest" GRCOV_ARTIFACT = "public/build/grcov.tar.xz" +logger = logging.getLogger(__name__) + def is_taskcluster_loaner(): return "TASKCLUSTER_INTERACTIVE" in os.environ @@ -311,6 +317,27 @@ def download_grcov(): return local_path +def upload_html_report( + report_dir, base_artifact="public/report", ttl=timedelta(days=10) +): + assert os.path.isdir(report_dir), "Not a directory {}".format(report_dir) + report_dir = os.path.realpath(report_dir) + assert not base_artifact.endswith("/"), "No trailing / in base_artifact" + + # Use Taskcluster proxy when available + taskcluster.auth() + + for path in Path(report_dir).rglob("*"): + + filename = str(path.relative_to(report_dir)) + content_type = magic.from_file(str(path), mime=True) + logger.debug("Uploading {} as {}".format(filename, content_type)) + + taskcluster.upload_artifact( + "{}/{}".format(base_artifact, filename), path.read_text(), content_type, ttl + ) + + def main(): parser = argparse.ArgumentParser() @@ -438,6 +465,9 @@ def main(): else: generate_report(grcov_path, "html", args.output_dir, artifact_paths) + if is_taskcluster_loaner(): + upload_html_report(args.output_dir) + if __name__ == "__main__": main() diff --git a/report/requirements.txt b/report/requirements.txt index c8fd962f4..fcb8f67af 100644 --- a/report/requirements.txt +++ b/report/requirements.txt @@ -1,2 +1,3 @@ -taskcluster==24.2.0 +python-magic==0.4.15 +taskcluster==24.3.1 tenacity==6.0.0 diff --git a/report/setup.py b/report/setup.py index 1630e85b0..8b11f4138 100644 --- a/report/setup.py +++ b/report/setup.py @@ -12,7 +12,11 @@ def read_requirements(file_): for line in f.readlines(): line = line.strip() if line.startswith("https://"): - line = line.split("#")[1].split("egg=")[1] + params = { + p[: p.index("=")]: p[p.index("=") + 1 :] + for p in line.split("#")[1].split("&") + } + line = params["egg"] elif line == "" or line.startswith("#") or line.startswith("-"): continue line = line.split("#")[0].strip() diff --git a/report/tests/test.py b/report/tests/test.py index 0682c3325..48fb555ab 100644 --- a/report/tests/test.py +++ b/report/tests/test.py @@ -7,7 +7,9 @@ import errno import os import shutil +import tempfile import unittest +from datetime import timedelta from firefox_code_coverage import codecoverage @@ -143,6 +145,19 @@ def test_download_grcov(self): with open("grcov_ver", "r") as f: self.assertEqual(ver, f.read()) + def test_upload_report(self): + + # Can only run on Taskcluster + if "TASK_ID" not in os.environ: + return + + _dir = tempfile.mkdtemp() + + with open(os.path.join(_dir, "report.html"), "w") as f: + f.write("This is a test") + + codecoverage.upload_html_report(str(_dir), ttl=timedelta(days=1)) + if __name__ == "__main__": unittest.main()