report: Upload HTML report as artifact, fixes #388 (#395)

Author: La0

.isort.cfg

@@ -1,6 +1,6 @@
 [settings]
 known_first_party = code_coverage_backend,code_coverage_bot,code_coverage_events,code_coverage_tools,conftest,firefox_code_coverage
-known_third_party = connexion,datadog,dateutil,fakeredis,flask,flask_cors,flask_talisman,google,hglib,jsone,jsonschema,libmozdata,libmozevent,logbook,pytest,pytz,raven,redis,requests,responses,setuptools,structlog,taskcluster,tenacity,werkzeug,zstandard
+known_third_party = connexion,datadog,dateutil,fakeredis,flask,flask_cors,flask_talisman,google,hglib,jsone,jsonschema,libmozdata,libmozevent,logbook,magic,pytest,pytz,raven,redis,requests,responses,setuptools,structlog,taskcluster,tenacity,werkzeug,zstandard
 force_single_line = True
 default_section=FIRSTPARTY
 line_length=159

.taskcluster.yml

@@ -143,13 +143,15 @@ tasks:
       created: {$fromNow: ''}
       deadline: {$fromNow: '1 hour'}
       payload:
+        features:
+          taskclusterProxy: true
         maxRunTime: 3600
         image: python:3.8
         command:
           - sh
           - -lxce
           - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks &&
-            cd /src/report && pip install -r requirements.txt && ./ci-test.sh"
+            cd /src/report && ./ci-test.sh"
       metadata:
         name: "Code Coverage Report checks: unit tests"
         description: Check python code with unittest

report/ci-test.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -e
 
-pip install --disable-pip-version-check --no-cache-dir --quiet -r test-requirements.txt
+pip install --disable-pip-version-check --no-cache-dir --quiet -r test-requirements.txt -r requirements.txt .
 
 python -m unittest discover tests
 python setup.py sdist bdist_wheel

report/firefox_code_coverage/codecoverage.py

@@ -3,6 +3,7 @@
 import argparse
 import errno
 import json
+import logging
 import os
 import shutil
 import subprocess
@@ -11,7 +12,10 @@
 import tempfile
 import time
 import warnings
+from datetime import timedelta
+from pathlib import Path
 
+import magic
 import requests
 import tenacity
 
@@ -24,6 +28,8 @@
 GRCOV_INDEX = "gecko.cache.level-3.toolchains.v3.linux64-grcov.latest"
 GRCOV_ARTIFACT = "public/build/grcov.tar.xz"
 
+logger = logging.getLogger(__name__)
+
 
 def is_taskcluster_loaner():
     return "TASKCLUSTER_INTERACTIVE" in os.environ
@@ -311,6 +317,27 @@ def download_grcov():
     return local_path
 
 
+def upload_html_report(
+    report_dir, base_artifact="public/report", ttl=timedelta(days=10)
+):
+    assert os.path.isdir(report_dir), "Not a directory {}".format(report_dir)
+    report_dir = os.path.realpath(report_dir)
+    assert not base_artifact.endswith("/"), "No trailing / in base_artifact"
+
+    # Use Taskcluster proxy when available
+    taskcluster.auth()
+
+    for path in Path(report_dir).rglob("*"):
+
+        filename = str(path.relative_to(report_dir))
+        content_type = magic.from_file(str(path), mime=True)
+        logger.debug("Uploading {} as {}".format(filename, content_type))
+
+        taskcluster.upload_artifact(
+            "{}/{}".format(base_artifact, filename), path.read_text(), content_type, ttl
+        )
+
+
 def main():
     parser = argparse.ArgumentParser()
 
@@ -438,6 +465,9 @@ def main():
     else:
         generate_report(grcov_path, "html", args.output_dir, artifact_paths)
 
+        if is_taskcluster_loaner():
+            upload_html_report(args.output_dir)
+
 
 if __name__ == "__main__":
     main()

report/requirements.txt

@@ -1,2 +1,3 @@
-taskcluster==24.2.0
+python-magic==0.4.15
+taskcluster==24.3.1
 tenacity==6.0.0

report/setup.py

@@ -12,7 +12,11 @@ def read_requirements(file_):
         for line in f.readlines():
             line = line.strip()
             if line.startswith("https://"):
-                line = line.split("#")[1].split("egg=")[1]
+                params = {
+                    p[: p.index("=")]: p[p.index("=") + 1 :]
+                    for p in line.split("#")[1].split("&")
+                }
+                line = params["egg"]
             elif line == "" or line.startswith("#") or line.startswith("-"):
                 continue
             line = line.split("#")[0].strip()

report/tests/test.py

@@ -7,7 +7,9 @@
 import errno
 import os
 import shutil
+import tempfile
 import unittest
+from datetime import timedelta
 
 from firefox_code_coverage import codecoverage
 
@@ -143,6 +145,19 @@ def test_download_grcov(self):
         with open("grcov_ver", "r") as f:
             self.assertEqual(ver, f.read())
 
+    def test_upload_report(self):
+
+        # Can only run on Taskcluster
+        if "TASK_ID" not in os.environ:
+            return
+
+        _dir = tempfile.mkdtemp()
+
+        with open(os.path.join(_dir, "report.html"), "w") as f:
+            f.write("<strong>This is a test</strong>")
+
+        codecoverage.upload_html_report(str(_dir), ttl=timedelta(days=1))
+
 
 if __name__ == "__main__":
     unittest.main()