Skip to content

Commit 4e49abe

Browse files
mbhavembhave
committed
Trigger cache update only for changed projects
Closes gh-23
1 parent aed4f64 commit 4e49abe

File tree

8 files changed

+254
-25
lines changed

8 files changed

+254
-25
lines changed

‎src/main/java/io/spring/projectapi/ProjectRepository.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
*/
3232
public interface ProjectRepository {
3333

34-
void update();
34+
void update(List<String> changes);
3535

3636
Collection<Project> getProjects();
3737

‎src/main/java/io/spring/projectapi/github/GithubProjectRepository.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ class GithubProjectRepository implements ProjectRepository {
4242
}
4343

4444
@Override
45-
public void update() {
46-
this.projectData = ProjectData.load(this.githubQueries);
45+
public void update(List<String> changes) {
46+
this.projectData = ProjectData.update(this.projectData, changes, this.githubQueries);
4747
}
4848

4949
@Override

‎src/main/java/io/spring/projectapi/github/GithubQueries.java‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
import java.util.LinkedHashMap;
2222
import java.util.List;
2323
import java.util.Map;
24+
import java.util.regex.Matcher;
25+
import java.util.regex.Pattern;
2426

2527
import com.fasterxml.jackson.core.JsonProcessingException;
2628
import com.fasterxml.jackson.core.type.TypeReference;
@@ -33,7 +35,9 @@
3335
import org.springframework.core.ParameterizedTypeReference;
3436
import org.springframework.http.RequestEntity;
3537
import org.springframework.http.ResponseEntity;
38+
import org.springframework.util.Assert;
3639
import org.springframework.util.StringUtils;
40+
import org.springframework.web.client.HttpClientErrorException;
3741
import org.springframework.web.client.RestTemplate;
3842

3943
/**
@@ -59,6 +63,8 @@ public class GithubQueries {
5963

6064
private static final String DEFAULT_SUPPORT_POLICY = "SPRING_BOOT";
6165

66+
private static final Pattern PROJECT_FILE = Pattern.compile("project\\/(.*)\\/.*");
67+
6268
private static final ParameterizedTypeReference<Map<String, Object>> STRING_OBJECT_MAP = new ParameterizedTypeReference<>() {
6369
};
6470

@@ -96,6 +102,95 @@ ProjectData getData() {
96102
return new ProjectData(projects, documentation, support, supportPolicy);
97103
}
98104

105+
ProjectData updateData(ProjectData data, List<String> changes) {
106+
Assert.notNull(data, "Project data should not be null");
107+
Map<String, Project> projects = new LinkedHashMap<>(data.project());
108+
Map<String, List<ProjectDocumentation>> documentation = new LinkedHashMap<>(data.documentation());
109+
Map<String, List<ProjectSupport>> support = new LinkedHashMap<>(data.support());
110+
Map<String, String> supportPolicy = new LinkedHashMap<>(data.supportPolicy());
111+
Map<String, Boolean> checkedProjects = new LinkedHashMap<>();
112+
try {
113+
changes.forEach((change) -> {
114+
ProjectFile file = ProjectFile.from(change);
115+
if (ProjectFile.OTHER.equals(file)) {
116+
return;
117+
}
118+
updateData(change, file, projects, supportPolicy, documentation, support, checkedProjects);
119+
});
120+
}
121+
catch (Exception ex) {
122+
logger.debug("Could not update data due to '%s'".formatted(ex.getMessage()));
123+
}
124+
return new ProjectData(projects, documentation, support, supportPolicy);
125+
}
126+
127+
private void updateData(String change, ProjectFile file, Map<String, Project> projects,
128+
Map<String, String> supportPolicy, Map<String, List<ProjectDocumentation>> documentation,
129+
Map<String, List<ProjectSupport>> support, Map<String, Boolean> checkedprojects) {
130+
Matcher matcher = PROJECT_FILE.matcher(change);
131+
if (!matcher.matches()) {
132+
return;
133+
}
134+
String slug = matcher.group(1);
135+
if (checkedprojects.get(slug) == null) {
136+
checkedprojects.put(slug, doesProjectExist(slug));
137+
}
138+
if (checkedprojects.get(slug)) {
139+
updateFromIndex(file, projects, supportPolicy, slug);
140+
updateDocumentation(file, documentation, slug);
141+
updateSupport(file, support, slug);
142+
return;
143+
}
144+
projects.remove(slug);
145+
documentation.remove(slug);
146+
support.remove(slug);
147+
supportPolicy.remove(slug);
148+
}
149+
150+
private void updateSupport(ProjectFile file, Map<String, List<ProjectSupport>> support, String slug) {
151+
if (ProjectFile.SUPPORT.equals(file)) {
152+
List<ProjectSupport> projectSupports = getProjectSupports(slug);
153+
support.put(slug, projectSupports);
154+
}
155+
}
156+
157+
private void updateDocumentation(ProjectFile file, Map<String, List<ProjectDocumentation>> documentation,
158+
String slug) {
159+
if (ProjectFile.DOCUMENTATION.equals(file)) {
160+
List<ProjectDocumentation> projectDocumentation = getProjectDocumentations(slug);
161+
documentation.put(slug, projectDocumentation);
162+
}
163+
}
164+
165+
private void updateFromIndex(ProjectFile file, Map<String, Project> projects, Map<String, String> supportPolicy,
166+
String slug) {
167+
if (ProjectFile.INDEX.equals(file)) {
168+
ResponseEntity<Map<String, Object>> response = getFile(slug, "index.md");
169+
Project project = getProject(response, slug);
170+
if (project != null) {
171+
projects.put(slug, project);
172+
}
173+
String policy = getProjectSupportPolicy(response, slug);
174+
supportPolicy.put(slug, policy);
175+
}
176+
}
177+
178+
private boolean doesProjectExist(String projectSlug) {
179+
RequestEntity<Void> request = RequestEntity.get("/project/{projectSlug}?ref=" + this.branch, projectSlug)
180+
.build();
181+
try {
182+
this.restTemplate.exchange(request, STRING_OBJECT_MAP);
183+
}
184+
catch (Exception ex) {
185+
if (ex instanceof HttpClientErrorException) {
186+
if (((HttpClientErrorException) ex).getStatusCode().value() == 404) {
187+
return false;
188+
}
189+
}
190+
}
191+
return true;
192+
}
193+
99194
private void populateData(Map<String, Object> project, Map<String, Project> projects,
100195
Map<String, List<ProjectDocumentation>> documentation, Map<String, List<ProjectSupport>> support,
101196
Map<String, String> supportPolicy) {
@@ -188,4 +283,29 @@ private String getFileContent(ResponseEntity<Map<String, Object>> exchange) {
188283
return new String(contents);
189284
}
190285

286+
enum ProjectFile {
287+
288+
INDEX,
289+
290+
SUPPORT,
291+
292+
DOCUMENTATION,
293+
294+
OTHER;
295+
296+
static ProjectFile from(String fileName) {
297+
if (fileName.contains("index.md")) {
298+
return INDEX;
299+
}
300+
if (fileName.contains("documentation.json")) {
301+
return DOCUMENTATION;
302+
}
303+
if (fileName.contains("support.json")) {
304+
return SUPPORT;
305+
}
306+
return OTHER;
307+
}
308+
309+
}
310+
191311
}

‎src/main/java/io/spring/projectapi/github/ProjectData.java‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import java.util.List;
2020
import java.util.Map;
2121

22+
import org.jetbrains.annotations.NotNull;
23+
2224
/**
2325
* Represents cached data from Github.
2426
*
@@ -34,10 +36,20 @@ record ProjectData(Map<String, Project> project, Map<String, List<ProjectDocumen
3436

3537
public static ProjectData load(GithubQueries githubQueries) {
3638
ProjectData data = githubQueries.getData();
37-
Map<String, Project> projects = data.project();
38-
Map<String, List<ProjectDocumentation>> documentation = data.documentation();
39-
Map<String, List<ProjectSupport>> support = data.support();
40-
Map<String, String> supportPolicy = data.supportPolicy();
39+
return getImmutableProjectData(data);
40+
}
41+
42+
public static ProjectData update(ProjectData data, List<String> changes, GithubQueries githubQueries) {
43+
ProjectData updatedData = githubQueries.updateData(data, changes);
44+
return getImmutableProjectData(updatedData);
45+
}
46+
47+
@NotNull
48+
private static ProjectData getImmutableProjectData(ProjectData updatedData) {
49+
Map<String, Project> projects = updatedData.project();
50+
Map<String, List<ProjectDocumentation>> documentation = updatedData.documentation();
51+
Map<String, List<ProjectSupport>> support = updatedData.support();
52+
Map<String, String> supportPolicy = updatedData.supportPolicy();
4153
return new ProjectData(Map.copyOf(projects), Map.copyOf(documentation), Map.copyOf(support),
4254
Map.copyOf(supportPolicy));
4355
}

‎src/main/java/io/spring/projectapi/web/webhook/CacheController.java‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
import java.nio.charset.StandardCharsets;
2222
import java.security.InvalidKeyException;
2323
import java.security.NoSuchAlgorithmException;
24+
import java.util.ArrayList;
25+
import java.util.List;
2426
import java.util.Map;
2527

2628
import javax.crypto.Mac;
@@ -87,6 +89,7 @@ private void verifyHmacSignature(String message, String signature) {
8789
}
8890

8991
@PostMapping("/refresh_cache")
92+
@SuppressWarnings("unchecked")
9093
public ResponseEntity<String> refresh(@RequestBody String payload,
9194
@RequestHeader("X-Hub-Signature") String signature,
9295
@RequestHeader(name = "X-GitHub-Event", required = false, defaultValue = "push") String event)
@@ -97,10 +100,26 @@ public ResponseEntity<String> refresh(@RequestBody String payload,
97100
}
98101
Map<?, ?> push = this.objectMapper.readValue(payload, Map.class);
99102
logPayload(push);
100-
this.repository.update();
103+
List<Map<String, ?>> commits = (List<Map<String, ?>>) push.get("commits");
104+
List<String> changes = getChangedFiles(commits);
105+
this.repository.update(changes);
101106
return ResponseEntity.ok("{ \"message\": \"Successfully processed cache refresh\" }");
102107
}
103108

109+
@SuppressWarnings("unchecked")
110+
private static List<String> getChangedFiles(List<Map<String, ?>> commits) {
111+
List<String> changedFiles = new ArrayList<>();
112+
commits.forEach((commit) -> {
113+
List<String> added = (List<String>) commit.get("added");
114+
List<String> removed = (List<String>) commit.get("removed");
115+
List<String> modified = (List<String>) commit.get("modified");
116+
changedFiles.addAll(added);
117+
changedFiles.addAll(removed);
118+
changedFiles.addAll(modified);
119+
});
120+
return changedFiles.stream().distinct().toList();
121+
}
122+
104123
@ExceptionHandler(WebhookAuthenticationException.class)
105124
public ResponseEntity<String> handleWebhookAuthenticationFailure(WebhookAuthenticationException exception) {
106125
logger.error("Webhook authentication failure: " + exception.getMessage());

‎src/test/java/io/spring/projectapi/github/GithubProjectRepositoryTests.java‎

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,11 @@
2424
import io.spring.projectapi.github.Project.Status;
2525
import org.junit.jupiter.api.BeforeEach;
2626
import org.junit.jupiter.api.Test;
27-
import org.mockito.Mockito;
28-
import org.mockito.verification.VerificationMode;
2927

3028
import static org.assertj.core.api.Assertions.assertThat;
3129
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
30+
import static org.mockito.ArgumentMatchers.any;
3231
import static org.mockito.BDDMockito.given;
33-
import static org.mockito.Mockito.atMostOnce;
3432
import static org.mockito.Mockito.mock;
3533
import static org.mockito.Mockito.verify;
3634

@@ -43,27 +41,32 @@ class GithubProjectRepositoryTests {
4341

4442
private GithubQueries githubQueries;
4543

44+
private ProjectData data;
45+
4646
@BeforeEach
4747
void setup() {
4848
this.githubQueries = mock(GithubQueries.class);
49-
setupGithubResponse("spring-boot");
49+
this.data = getData("spring-boot");
50+
given(this.githubQueries.getData()).willReturn(this.data);
5051
this.projectRepository = new GithubProjectRepository(this.githubQueries);
5152
}
5253

5354
@Test
5455
void dataLoadedOnBeanCreation() {
5556
validateCachedValues("spring-boot");
56-
verifyCacheUpdate(atMostOnce());
57+
verify(this.githubQueries).getData();
5758
}
5859

5960
@Test
6061
void updateRefreshesCache() {
61-
setupGithubResponse("spring-boot-updated");
62-
this.projectRepository.update();
62+
List<String> changes = List.of("project/spring-boot-updated/index.md",
63+
"project/spring-boot-updated/documentation.json", "project/spring-boot-updated/support.json");
64+
given(this.githubQueries.updateData(any(), any())).willReturn(getData("spring-boot-updated"));
65+
this.projectRepository.update(changes);
6366
assertThatExceptionOfType(NoSuchGithubProjectException.class)
6467
.isThrownBy(() -> this.projectRepository.getProject("spring-boot"));
6568
validateCachedValues("spring-boot-updated");
66-
verifyCacheUpdate(Mockito.atMost(2));
69+
verify(this.githubQueries).updateData(this.data, changes);
6770
}
6871

6972
@Test
@@ -133,14 +136,6 @@ private void validateCachedValues(String projectSlug) {
133136
assertThat(policy).isEqualTo("UPSTREAM");
134137
}
135138

136-
private void setupGithubResponse(String project) {
137-
given(this.githubQueries.getData()).willReturn(getData(project));
138-
}
139-
140-
private void verifyCacheUpdate(VerificationMode mode) {
141-
verify(this.githubQueries, mode).getData();
142-
}
143-
144139
private ProjectData getData(String project) {
145140
return new ProjectData(getProjects(project), getProjectDocumentation(project), getProjectSupports(project),
146141
getProjectSupportPolicy(project));

0 commit comments

Comments
 (0)