Skip to content

Commit 2b1e616

Browse files
author
Bastien Abadie
committed
backend: Ingest only first report in a push to avoid timeouts
1 parent ffc7c1b commit 2b1e616

File tree

2 files changed

+21
-12
lines changed

2 files changed

+21
-12
lines changed

backend/code_coverage_backend/gcp.py

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -78,40 +78,47 @@ def __init__(self, reports_dir=None):
7878
for report in self.list_reports(repo, nb=1):
7979
self.download_report(report)
8080

81-
def ingest_pushes(self, repository, min_push_id=None, nb_pages=3):
81+
def ingest_pushes(self, repository, platform, suite, min_push_id=None, nb_pages=3):
8282
"""
8383
Ingest HGMO changesets and pushes into our Redis Cache
8484
The pagination goes from oldest to newest, starting from the optional min_push_id
8585
"""
86+
ingested = False
8687
for push_id, push in hgmo_pushes(repository, min_push_id, nb_pages):
8788
for changeset in push["changesets"]:
8889
# TODO: look all neighboring reports on GCP
8990
report = Report(
9091
self.reports_dir,
9192
repository,
9293
changeset,
94+
platform,
95+
suite,
9396
push_id=push_id,
9497
date=push["date"],
9598
)
96-
if self.ingest_report(report):
99+
100+
# Always link changeset to push to find closest available report
101+
self.redis.hmset(
102+
KEY_CHANGESET.format(
103+
repository=report.repository, changeset=report.changeset
104+
),
105+
{"push": report.push_id, "date": report.date},
106+
)
107+
108+
if not ingested and self.ingest_report(report):
97109
logger.info(
98110
"Found report in that push", push_id=push_id, report=str(report)
99111
)
100112

113+
# Only ingest first report found in a push in order to stay below 30s response time
114+
ingested = True
115+
101116
def ingest_report(self, report):
102117
"""
103118
When a report exist for a changeset, download it and update redis data
104119
"""
105120
assert isinstance(report, Report)
106121

107-
# Always link changeset to push to find closest available report
108-
self.redis.hmset(
109-
KEY_CHANGESET.format(
110-
repository=report.repository, changeset=report.changeset
111-
),
112-
{"push": report.push_id, "date": report.date},
113-
)
114-
115122
# Download the report
116123
if not self.download_report(report):
117124
logger.info("Report not available", report=str(report))
@@ -228,7 +235,9 @@ def find_closest_report(
228235
push_id, _ = hgmo_revision_details(repository, changeset)
229236

230237
# Ingest pushes as we clearly don't have it in cache
231-
self.ingest_pushes(repository, min_push_id=push_id - 1, nb_pages=1)
238+
self.ingest_pushes(
239+
repository, platform, suite, min_push_id=push_id - 1, nb_pages=1
240+
)
232241

233242
# Load report from that push
234243
return self.find_report(

backend/tests/test_gcp.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ def test_ingest_hgmo(mock_cache, mock_hgmo):
122122
# Ingest last pushes
123123
assert mock_cache.list_reports("myrepo") == []
124124
assert len(mock_cache.redis.keys("changeset:myrepo:*")) == 0
125-
mock_cache.ingest_pushes("myrepo")
125+
mock_cache.ingest_pushes("myrepo", "all", "all")
126126
assert len(mock_cache.redis.keys("changeset:myrepo:*")) > 0
127127
assert mock_cache.list_reports("myrepo") == [
128128
Report(mock_cache.reports_dir, "myrepo", rev, push_id=1, date=995)

0 commit comments

Comments
 (0)