bot: Build then upload all reports

Author: Bastien Abadie

bot/code_coverage_bot/artifacts.py

@@ -44,12 +44,14 @@ def get_chunks(self, platform):
         )
 
     def get_suites(self):
+        # Add the full report
+        out = collections.defaultdict(list)
+        out[("all", "all")] = [artifact.path for artifact in self.artifacts]
+
         # Group by suite first
         suites = itertools.groupby(
             sorted(self.artifacts, key=lambda a: a.suite), lambda a: a.suite
         )
-
-        out = collections.defaultdict(list)
         for suite, artifacts in suites:
             artifacts = list(artifacts)
 

bot/code_coverage_bot/codecov.py

@@ -42,6 +42,7 @@ def __init__(self, repository, revision, task_name_filter, cache_root):
 
         temp_dir = tempfile.mkdtemp()
         self.artifacts_dir = os.path.join(temp_dir, "ccov-artifacts")
+        self.reports_dir = os.path.join(temp_dir, "ccov-reports")
 
         self.index_service = taskcluster_config.get_service("index")
 
@@ -118,23 +119,18 @@ def retrieve_source_and_artifacts(self):
             # Thread 2 - Clone repository.
             executor.submit(self.clone_repository, self.repository, self.revision)
 
-    def generate_covdir(self):
+    def build_reports(self, only=None):
         """
-        Build the full covdir report using current artifacts
+        Build all the possible covdir reports using current artifacts
         """
-        output = grcov.report(
-            self.artifactsHandler.get(), source_dir=self.repo_dir, out_format="covdir"
-        )
-        logger.info("Covdir report generated successfully")
-        return json.loads(output)
+        os.makedirs(self.reports_dir, exist_ok=True)
 
-    def build_suites(self):
-        """
-        Build all the detailed covdir reports using current artifacts
-        and upload them directly on GCP
-        """
+        reports = {}
         for (platform, suite), artifacts in self.artifactsHandler.get_suites().items():
 
+            if only is not None and (platform, suite) not in only:
+                continue
+
             # Generate covdir report for that suite & platform
             logger.info(
                 "Building covdir suite report",
@@ -146,27 +142,33 @@ def build_suites(self):
                 artifacts, source_dir=self.repo_dir, out_format="covdir"
             )
 
-            # Then upload on GCP
-            report = json.loads(output)
+            # Write output on FS
+            path = os.path.join(self.reports_dir, f"{platform}.{suite}.json")
+            with open(path, "wb") as f:
+                f.write(output)
+
+            reports[(platform, suite)] = path
+
+        return reports
+
+    def upload_reports(self, reports):
+        """
+        Upload all provided covdir reports on GCP
+        """
+        for (platform, suite), path in reports.items():
+            report = json.load(open(path))
             uploader.gcp(
                 self.branch, self.revision, report, suite=suite, platform=platform
             )
 
-    # This function is executed when the bot is triggered at the end of a mozilla-central build.
-    def go_from_trigger_mozilla_central(self):
-        # Check the covdir report does not already exists
-        if uploader.gcp_covdir_exists(self.branch, self.revision, "full"):
-            logger.warn("Covdir report already on GCP")
-            return
-
-        self.retrieve_source_and_artifacts()
-
-        # Check that all JavaScript files present in the coverage artifacts actually exist.
-        # If they don't, there might be a bug in the LCOV rewriter.
+    def check_javascript_files(self):
+        """
+        Check that all JavaScript files present in the coverage artifacts actually exist.
+        If they don't, there might be a bug in the LCOV rewriter.
+        """
         for artifact in self.artifactsHandler.get():
             if "jsvm" not in artifact:
                 continue
-
             with zipfile.ZipFile(artifact, "r") as zf:
                 for file_name in zf.namelist():
                     with zf.open(file_name, "r") as fl:
@@ -185,7 +187,25 @@ def go_from_trigger_mozilla_central(self):
                                 f"{missing_files} are present in coverage reports, but missing from the repository"
                             )
 
-        report = self.generate_covdir()
+    # This function is executed when the bot is triggered at the end of a mozilla-central build.
+    def go_from_trigger_mozilla_central(self):
+        # Check the covdir report does not already exists
+        if uploader.gcp_covdir_exists(self.branch, self.revision, "all", "all"):
+            logger.warn("Full covdir report already on GCP")
+            return
+
+        self.retrieve_source_and_artifacts()
+
+        # TODO: restore that check
+        # self.check_javascript_files()
+
+        reports = self.build_reports()
+        logger.info("Built all covdir reports", nb=len(reports))
+
+        # Retrieve the full report
+        full_path = reports.get(("all", "all"))
+        assert full_path is not None, "Missing full report (all:all)"
+        report = json.load(open(full_path))
 
         paths = uploader.covdir_paths(report)
         expected_extensions = [".js", ".cpp"]
@@ -194,6 +214,9 @@ def go_from_trigger_mozilla_central(self):
                 path.endswith(extension) for path in paths
             ), "No {} file in the generated report".format(extension)
 
+        self.upload_reports(reports)
+        logger.info("Uploaded all covdir reports", nb=len(reports))
+
         # Get pushlog and ask the backend to generate the coverage by changeset
         # data, which will be cached.
         with hgmo.HGMO(self.repo_dir) as hgmo_server:
@@ -203,10 +226,6 @@ def go_from_trigger_mozilla_central(self):
         phabricatorUploader = PhabricatorUploader(self.repo_dir, self.revision)
         changesets_coverage = phabricatorUploader.upload(report, changesets)
 
-        uploader.gcp(self.branch, self.revision, report)
-        logger.info("Main Build uploaded on GCP")
-
-        self.build_suites()
         notify_email(self.revision, changesets, changesets_coverage)
 
     # This function is executed when the bot is triggered at the end of a try build.
@@ -226,7 +245,10 @@ def go_from_trigger_try(self):
 
         self.retrieve_source_and_artifacts()
 
-        report = self.generate_covdir()
+        reports = self.build_reports(only=("all", "all"))
+        full_path = reports.get(("all", "all"))
+        assert full_path is not None, "Missing full report (all:all)"
+        report = json.load(open(full_path))
 
         logger.info("Upload changeset coverage data to Phabricator")
         phabricatorUploader.upload(report, changesets)

bot/code_coverage_bot/uploader.py

@@ -12,30 +12,29 @@
 from code_coverage_tools.gcp import get_bucket
 
 logger = structlog.get_logger(__name__)
-GCP_COVDIR_PATH = "{repository}/{revision}/{name}.json.zstd"
+GCP_COVDIR_PATH = "{repository}/{revision}/{platform}:{suite}.json.zstd"
 
 
-def gcp(repository, revision, report, platform=None, suite=None):
+def gcp(repository, revision, report, platform, suite):
     """
     Upload a grcov raw report on Google Cloud Storage
     * Compress with zstandard
     * Upload on bucket using revision in name
     * Trigger ingestion on channel's backend
     """
     assert isinstance(report, dict)
+    assert isinstance(platform, str)
+    assert isinstance(suite, str)
     bucket = get_bucket(secrets[secrets.GOOGLE_CLOUD_STORAGE])
 
     # Compress report
     compressor = zstd.ZstdCompressor()
     archive = compressor.compress(json.dumps(report).encode("utf-8"))
 
     # Upload archive
-    if platform and suite:
-        name = f"{platform}:{suite}"
-    else:
-        name = "full"
-
-    path = GCP_COVDIR_PATH.format(repository=repository, revision=revision, name=name)
+    path = GCP_COVDIR_PATH.format(
+        repository=repository, revision=revision, platform=platform, suite=suite
+    )
     blob = bucket.blob(path)
     blob.upload_from_string(archive)
 
@@ -56,12 +55,14 @@ def gcp(repository, revision, report, platform=None, suite=None):
     return blob
 
 
-def gcp_covdir_exists(repository, revision, name):
+def gcp_covdir_exists(repository, revision, platform, suite):
     """
     Check if a covdir report exists on the Google Cloud Storage bucket
     """
     bucket = get_bucket(secrets[secrets.GOOGLE_CLOUD_STORAGE])
-    path = GCP_COVDIR_PATH.format(repository=repository, revision=revision, name=name)
+    path = GCP_COVDIR_PATH.format(
+        repository=repository, revision=revision, platform=platform, suite=suite
+    )
     blob = bucket.blob(path)
     return blob.exists()
 

bot/tests/test_artifacts.py

@@ -73,6 +73,17 @@ def add_dir(files):
     a = ArtifactsHandler([])
     a.artifacts = fake_artifacts
     assert dict(a.get_suites()) == {
+        ("all", "all"): add_dir(
+            [
+                "windows_mochitest-1_code-coverage-jsvm.info",
+                "linux_mochitest-2_code-coverage-grcov.zip",
+                "windows_xpcshell-7_code-coverage-jsvm.info",
+                "linux_xpcshell-7_code-coverage-grcov.zip",
+                "linux_xpcshell-3_code-coverage-grcov.zip",
+                "windows_cppunit_code-coverage-grcov.zip",
+                "linux_firefox-ui-functional-remote_code-coverage-jsvm.info",
+            ]
+        ),
         ("linux", "all"): add_dir(
             [
                 "linux_firefox-ui-functional-remote_code-coverage-jsvm.info",