Skip to content

Commit 93134c5

Browse files
patchwork01patchwork01
authored
Merge pull request #5571 from gchq/5559-remove-stopping-ecs-tasks-from-the-teardown-script
5559: Remove stopping ECS tasks from the teardown script
2 parents 972789f + 044fc35 commit 93134c5

File tree

6 files changed

+3
-187
lines changed

6 files changed

+3
-187
lines changed

‎java/clients/src/main/java/sleeper/clients/teardown/ShutdownSystemProcesses.java‎

Lines changed: 2 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -18,50 +18,40 @@
1818
import org.slf4j.Logger;
1919
import org.slf4j.LoggerFactory;
2020
import software.amazon.awssdk.services.cloudwatchevents.CloudWatchEventsClient;
21-
import software.amazon.awssdk.services.ecs.EcsClient;
2221
import software.amazon.awssdk.services.emr.EmrClient;
2322
import software.amazon.awssdk.services.emr.model.ListClustersResponse;
2423
import software.amazon.awssdk.services.emrserverless.EmrServerlessClient;
2524

2625
import sleeper.clients.deploy.PauseSystem;
2726
import sleeper.clients.util.EmrUtils;
28-
import sleeper.core.properties.SleeperProperties;
29-
import sleeper.core.properties.SleeperProperty;
3027
import sleeper.core.properties.instance.InstanceProperties;
3128
import sleeper.core.util.StaticRateLimit;
3229
import sleeper.core.util.ThreadSleep;
3330

3431
import java.util.List;
35-
import java.util.function.Consumer;
3632

37-
import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.BULK_EXPORT_CLUSTER;
38-
import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_CLUSTER;
39-
import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.INGEST_CLUSTER;
4033
import static sleeper.core.properties.instance.CommonProperty.ID;
41-
import static sleeper.core.util.RateLimitUtils.sleepForSustainedRatePerSecond;
4234

4335
public class ShutdownSystemProcesses {
4436

4537
private static final Logger LOGGER = LoggerFactory.getLogger(ShutdownSystemProcesses.class);
4638

4739
private final CloudWatchEventsClient cloudWatch;
48-
private final EcsClient ecs;
4940
private final EmrClient emrClient;
5041
private final EmrServerlessClient emrServerlessClient;
5142
private final StaticRateLimit<ListClustersResponse> listActiveClustersLimit;
5243
private final ThreadSleep threadSleep;
5344

5445
public ShutdownSystemProcesses(TearDownClients clients) {
55-
this(clients.getCloudWatch(), clients.getEcs(), clients.getEmr(), clients.getEmrServerless(), EmrUtils.LIST_ACTIVE_CLUSTERS_LIMIT, Thread::sleep);
46+
this(clients.getCloudWatch(), clients.getEmr(), clients.getEmrServerless(), EmrUtils.LIST_ACTIVE_CLUSTERS_LIMIT, Thread::sleep);
5647
}
5748

5849
public ShutdownSystemProcesses(
59-
CloudWatchEventsClient cloudWatch, EcsClient ecs,
50+
CloudWatchEventsClient cloudWatch,
6051
EmrClient emrClient, EmrServerlessClient emrServerlessClient,
6152
StaticRateLimit<ListClustersResponse> listActiveClustersLimit,
6253
ThreadSleep threadSleep) {
6354
this.cloudWatch = cloudWatch;
64-
this.ecs = ecs;
6555
this.emrClient = emrClient;
6656
this.emrServerlessClient = emrServerlessClient;
6757
this.listActiveClustersLimit = listActiveClustersLimit;
@@ -72,18 +62,10 @@ public void shutdown(InstanceProperties instanceProperties, List<String> extraEC
7262
LOGGER.info("Shutting down system processes for instance {}", instanceProperties.get(ID));
7363
LOGGER.info("Pausing the system");
7464
PauseSystem.pause(cloudWatch, instanceProperties);
75-
stopECSTasks(instanceProperties, extraECSClusters);
7665
stopEMRClusters(instanceProperties);
7766
stopEMRServerlessApplication(instanceProperties);
7867
}
7968

80-
private void stopECSTasks(InstanceProperties instanceProperties, List<String> extraClusters) {
81-
stopTasks(ecs, instanceProperties, INGEST_CLUSTER);
82-
stopTasks(ecs, instanceProperties, COMPACTION_CLUSTER);
83-
stopTasks(ecs, instanceProperties, BULK_EXPORT_CLUSTER);
84-
extraClusters.forEach(clusterName -> stopTasks(ecs, clusterName));
85-
}
86-
8769
private void stopEMRClusters(InstanceProperties properties) throws InterruptedException {
8870
new TerminateEMRClusters(emrClient, properties.get(ID), listActiveClustersLimit, threadSleep).run();
8971
}
@@ -92,29 +74,4 @@ private void stopEMRServerlessApplication(InstanceProperties properties) throws
9274
new TerminateEMRServerlessApplications(emrServerlessClient, properties).run();
9375
}
9476

95-
public static <T extends SleeperProperty> void stopTasks(EcsClient ecs, SleeperProperties<T> properties, T property) {
96-
if (!properties.isSet(property)) {
97-
return;
98-
}
99-
stopTasks(ecs, properties.get(property));
100-
}
101-
102-
private static void stopTasks(EcsClient ecs, String clusterName) {
103-
LOGGER.info("Stopping tasks for ECS cluster {}", clusterName);
104-
forEachTaskArn(ecs, clusterName, taskArn -> {
105-
// Rate limit for ECS StopTask is 100 burst, 40 sustained:
106-
// https://docs.aws.amazon.com/AmazonECS/latest/APIReference/request-throttling.html
107-
sleepForSustainedRatePerSecond(30);
108-
ecs.stopTask(builder -> builder.cluster(clusterName).task(taskArn)
109-
.reason("Cleaning up before cdk destroy"));
110-
});
111-
}
112-
113-
private static void forEachTaskArn(EcsClient ecs, String clusterName, Consumer<String> consumer) {
114-
ecs.listTasksPaginator(builder -> builder.cluster(clusterName))
115-
.stream()
116-
.peek(response -> LOGGER.info("Found {} tasks", response.taskArns().size()))
117-
.flatMap(response -> response.taskArns().stream())
118-
.forEach(consumer);
119-
}
12077
}

‎java/clients/src/main/java/sleeper/clients/teardown/TearDownClients.java‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
import software.amazon.awssdk.services.cloudformation.CloudFormationClient;
2020
import software.amazon.awssdk.services.cloudwatchevents.CloudWatchEventsClient;
2121
import software.amazon.awssdk.services.ecr.EcrClient;
22-
import software.amazon.awssdk.services.ecs.EcsClient;
2322
import software.amazon.awssdk.services.emr.EmrClient;
2423
import software.amazon.awssdk.services.emrserverless.EmrServerlessClient;
2524
import software.amazon.awssdk.services.s3.S3Client;
@@ -31,7 +30,6 @@ public class TearDownClients {
3130

3231
private final S3Client s3;
3332
private final CloudWatchEventsClient cloudWatch;
34-
private final EcsClient ecs;
3533
private final EcrClient ecr;
3634
private final EmrClient emr;
3735
private final EmrServerlessClient emrServerless;
@@ -40,7 +38,6 @@ public class TearDownClients {
4038
private TearDownClients(Builder builder) {
4139
s3 = Objects.requireNonNull(builder.s3, "s3v2 must not be null");
4240
cloudWatch = Objects.requireNonNull(builder.cloudWatch, "cloudWatch must not be null");
43-
ecs = Objects.requireNonNull(builder.ecs, "ecs must not be null");
4441
ecr = Objects.requireNonNull(builder.ecr, "ecr must not be null");
4542
emr = Objects.requireNonNull(builder.emr, "emr must not be null");
4643
emrServerless = Objects.requireNonNull(builder.emrServerless, "emrServerless must not be null");
@@ -51,14 +48,12 @@ public static void withDefaults(TearDownOperation operation) throws IOException,
5148
try (S3Client s3Client = S3Client.create();
5249
CloudWatchEventsClient cloudWatchClient = CloudWatchEventsClient.create();
5350
EcrClient ecrClient = EcrClient.create();
54-
EcsClient ecsClient = EcsClient.create();
5551
EmrClient emrClient = EmrClient.create();
5652
EmrServerlessClient emrServerless = EmrServerlessClient.create();
5753
CloudFormationClient cloudFormationClient = CloudFormationClient.create()) {
5854
TearDownClients clients = builder()
5955
.s3(s3Client)
6056
.cloudWatch(cloudWatchClient)
61-
.ecs(ecsClient)
6257
.ecr(ecrClient)
6358
.emr(emrClient)
6459
.emrServerless(emrServerless)
@@ -80,10 +75,6 @@ public CloudWatchEventsClient getCloudWatch() {
8075
return cloudWatch;
8176
}
8277

83-
public EcsClient getEcs() {
84-
return ecs;
85-
}
86-
8778
public EcrClient getEcr() {
8879
return ecr;
8980
}
@@ -103,7 +94,6 @@ public CloudFormationClient getCloudFormation() {
10394
public static final class Builder {
10495
private S3Client s3;
10596
private CloudWatchEventsClient cloudWatch;
106-
private EcsClient ecs;
10797
private EcrClient ecr;
10898
private EmrClient emr;
10999
private EmrServerlessClient emrServerless;
@@ -122,11 +112,6 @@ public Builder cloudWatch(CloudWatchEventsClient cloudWatch) {
122112
return this;
123113
}
124114

125-
public Builder ecs(EcsClient ecs) {
126-
this.ecs = ecs;
127-
return this;
128-
}
129-
130115
public Builder ecr(EcrClient ecr) {
131116
this.ecr = ecr;
132117
return this;

‎java/clients/src/test/java/sleeper/clients/teardown/ShutdownSystemProcessesIT.java‎

Lines changed: 1 addition & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -33,23 +33,15 @@
3333
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
3434
import static com.github.tomakehurst.wiremock.client.WireMock.anyRequestedFor;
3535
import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl;
36-
import static com.github.tomakehurst.wiremock.client.WireMock.post;
3736
import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
3837
import static com.github.tomakehurst.wiremock.client.WireMock.verify;
3938
import static com.github.tomakehurst.wiremock.stubbing.Scenario.STARTED;
40-
import static sleeper.clients.testutil.ClientWiremockTestHelper.OPERATION_HEADER;
4139
import static sleeper.clients.testutil.ClientWiremockTestHelper.wiremockCloudWatchClient;
42-
import static sleeper.clients.testutil.ClientWiremockTestHelper.wiremockEcsClient;
4340
import static sleeper.clients.testutil.ClientWiremockTestHelper.wiremockEmrClient;
4441
import static sleeper.clients.testutil.ClientWiremockTestHelper.wiremockEmrServerlessClient;
4542
import static sleeper.clients.testutil.WiremockCloudWatchTestHelper.anyRequestedForCloudWatchEvents;
4643
import static sleeper.clients.testutil.WiremockCloudWatchTestHelper.disableRuleRequest;
4744
import static sleeper.clients.testutil.WiremockCloudWatchTestHelper.disableRuleRequestedFor;
48-
import static sleeper.clients.testutil.WiremockEcsTestHelper.MATCHING_LIST_TASKS_OPERATION;
49-
import static sleeper.clients.testutil.WiremockEcsTestHelper.MATCHING_STOP_TASK_OPERATION;
50-
import static sleeper.clients.testutil.WiremockEcsTestHelper.anyRequestedForEcs;
51-
import static sleeper.clients.testutil.WiremockEcsTestHelper.listTasksRequestedFor;
52-
import static sleeper.clients.testutil.WiremockEcsTestHelper.stopTaskRequestedFor;
5345
import static sleeper.clients.testutil.WiremockEmrServerlessTestHelper.aResponseWithApplicationWithNameAndState;
5446
import static sleeper.clients.testutil.WiremockEmrServerlessTestHelper.aResponseWithApplicationWithState;
5547
import static sleeper.clients.testutil.WiremockEmrServerlessTestHelper.aResponseWithJobRunWithState;
@@ -75,12 +67,10 @@
7567
import static sleeper.clients.testutil.WiremockEmrTestHelper.terminateJobFlowsRequestWithJobIdCount;
7668
import static sleeper.clients.testutil.WiremockEmrTestHelper.terminateJobFlowsRequestedFor;
7769
import static sleeper.clients.testutil.WiremockEmrTestHelper.terminateJobFlowsRequestedWithJobIdsCount;
78-
import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_CLUSTER;
7970
import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_JOB_CREATION_CLOUDWATCH_RULE;
8071
import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_TASK_CREATION_CLOUDWATCH_RULE;
8172
import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.GARBAGE_COLLECTOR_CLOUDWATCH_RULE;
8273
import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.INGEST_CLOUDWATCH_RULE;
83-
import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.INGEST_CLUSTER;
8474
import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.PARTITION_SPLITTING_CLOUDWATCH_RULE;
8575
import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.TABLE_METRICS_RULE;
8676
import static sleeper.core.properties.instance.CommonProperty.ID;
@@ -95,7 +85,7 @@ class ShutdownSystemProcessesIT {
9585

9686
@BeforeEach
9787
void setUp(WireMockRuntimeInfo runtimeInfo) {
98-
shutdown = new ShutdownSystemProcesses(wiremockCloudWatchClient(runtimeInfo), wiremockEcsClient(runtimeInfo),
88+
shutdown = new ShutdownSystemProcesses(wiremockCloudWatchClient(runtimeInfo),
9989
wiremockEmrClient(runtimeInfo), wiremockEmrServerlessClient(runtimeInfo), StaticRateLimit.none(), noWaits());
10090
}
10191

@@ -163,59 +153,6 @@ void shouldShutdownCloudWatchRulesWhenSet() throws Exception {
163153
}
164154
}
165155

166-
@Nested
167-
@DisplayName("Terminate running ECS tasks")
168-
class TerminateECSTasks {
169-
170-
@BeforeEach
171-
void setup() {
172-
properties.set(INGEST_CLUSTER, "test-ingest-cluster");
173-
stubFor(listActiveEmrClustersRequest()
174-
.willReturn(aResponseWithNoClusters()));
175-
stubFor(listActiveEmrApplicationsRequest()
176-
.willReturn(aResponseWithNoApplications()));
177-
}
178-
179-
@Test
180-
void shouldLookForECSTasksWhenClustersSet() throws Exception {
181-
// Given
182-
properties.set(COMPACTION_CLUSTER, "test-compaction-cluster");
183-
List<String> extraECSClusters = List.of("test-system-test-cluster");
184-
185-
stubFor(post("/")
186-
.withHeader(OPERATION_HEADER, MATCHING_LIST_TASKS_OPERATION)
187-
.willReturn(aResponse().withStatus(200).withBody("{\"nextToken\":null,\"taskArns\":[]}")));
188-
189-
// When
190-
shutdownWithExtraEcsClusters(extraECSClusters);
191-
192-
// Then
193-
verify(3, anyRequestedForEcs());
194-
verify(1, listTasksRequestedFor("test-ingest-cluster"));
195-
verify(1, listTasksRequestedFor("test-compaction-cluster"));
196-
verify(1, listTasksRequestedFor("test-system-test-cluster"));
197-
}
198-
199-
@Test
200-
void shouldStopECSTaskWhenOneIsFound() throws Exception {
201-
// Given
202-
stubFor(post("/")
203-
.withHeader(OPERATION_HEADER, MATCHING_LIST_TASKS_OPERATION)
204-
.willReturn(aResponse().withStatus(200).withBody("{\"nextToken\":null,\"taskArns\":[\"test-task\"]}")));
205-
stubFor(post("/")
206-
.withHeader(OPERATION_HEADER, MATCHING_STOP_TASK_OPERATION)
207-
.willReturn(aResponse().withStatus(200)));
208-
209-
// When
210-
shutdown();
211-
212-
// Then
213-
verify(2, anyRequestedForEcs());
214-
verify(1, listTasksRequestedFor("test-ingest-cluster"));
215-
verify(1, stopTaskRequestedFor("test-ingest-cluster", "test-task"));
216-
}
217-
}
218-
219156
@Nested
220157
@DisplayName("Terminate running EMR clusters")
221158
class TerminateEMRClusters {

‎java/clients/src/test/java/sleeper/clients/testutil/WiremockEcsTestHelper.java‎

Lines changed: 0 additions & 56 deletions
This file was deleted.

‎java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/TearDownMavenSystemTest.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,6 @@ public static void tearDown(
7070
instance.waitForStackToDelete();
7171
}
7272
for (TearDownSystemTestDeployment deployment : tearDownSystemTestDeployments) {
73-
deployment.shutdownSystemProcesses();
7473
deployment.deleteStack();
7574
}
7675
for (TearDownInstance instance : tearDownStandaloneInstances) {

‎java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/TearDownSystemTestDeployment.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121

2222
import sleeper.clients.teardown.RemoveECRRepositories;
2323
import sleeper.clients.teardown.RemoveJarsBucket;
24-
import sleeper.clients.teardown.ShutdownSystemProcesses;
2524
import sleeper.clients.teardown.TearDownClients;
2625
import sleeper.clients.teardown.WaitForStackToDelete;
2726
import sleeper.core.deploy.PopulateInstanceProperties;
@@ -31,7 +30,6 @@
3130
import java.io.IOException;
3231
import java.util.List;
3332

34-
import static sleeper.systemtest.configuration.SystemTestProperty.SYSTEM_TEST_CLUSTER_NAME;
3533
import static sleeper.systemtest.configuration.SystemTestProperty.SYSTEM_TEST_ID;
3634
import static sleeper.systemtest.configuration.SystemTestProperty.SYSTEM_TEST_JARS_BUCKET;
3735
import static sleeper.systemtest.configuration.SystemTestProperty.SYSTEM_TEST_REPO;
@@ -65,10 +63,6 @@ public void waitForStackToDelete() throws InterruptedException {
6563
WaitForStackToDelete.from(clients.getCloudFormation(), properties.get(SYSTEM_TEST_ID)).pollUntilFinished();
6664
}
6765

68-
public void shutdownSystemProcesses() throws InterruptedException {
69-
ShutdownSystemProcesses.stopTasks(clients.getEcs(), properties, SYSTEM_TEST_CLUSTER_NAME);
70-
}
71-
7266
public void cleanupAfterAllInstancesAndStackDeleted() throws InterruptedException, IOException {
7367
LOGGER.info("Removing the Jars bucket and docker containers");
7468
RemoveJarsBucket.remove(clients.getS3(), properties.get(SYSTEM_TEST_JARS_BUCKET));

0 commit comments

Comments
 (0)