Skip to content

Commit ba4a484

Browse files
james00012james00012
committed
fix: respect --timeout flag for AWS resource deletion waiters
AWS SDK waiters had hardcoded timeout values that ignored the user-specified --timeout flag. This adds a DefaultWaitTimeout constant (5 minutes) in base_resource.go and sets br.Timeout in PrepareContext. All resources with deletion waiters now use .Timeout instead of hardcoded values. Fixes #964
1 parent 4782605 commit ba4a484

File tree

13 files changed

+38
-36
lines changed

13 files changed

+38
-36
lines changed

‎aws/resources/asg.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package resources
22

33
import (
44
"context"
5-
"time"
65

76
"github.com/aws/aws-sdk-go-v2/aws"
87
"github.com/aws/aws-sdk-go-v2/service/autoscaling"
@@ -72,7 +71,7 @@ func (ag *ASGroups) nukeAll(groupNames []*string) error {
7271
waiter := autoscaling.NewGroupNotExistsWaiter(ag.Client)
7372
err := waiter.Wait(ag.Context, &autoscaling.DescribeAutoScalingGroupsInput{
7473
AutoScalingGroupNames: deletedGroupNames,
75-
}, 5*time.Minute)
74+
}, ag.Timeout)
7675

7776
if err != nil {
7877
logging.Errorf("[Failed] %s", err)

‎aws/resources/base_resource.go‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ const maxStopRetries = 3
1616
const waitDuration = 5 * time.Second
1717
const stopWaitDuration = 5 * time.Second
1818

19+
// DefaultWaitTimeout is the default timeout for AWS resource deletion waiters
20+
const DefaultWaitTimeout = 5 * time.Minute
21+
1922
// BaseAwsResource struct and its associated methods to serve as a placeholder or template for a resource that is not
2023
// yet fully implemented within a system or framework. Its purpose is to provide a skeleton structure that adheres to a
2124
// specific interface or contract expected by the system without containing the actual implementation details.
@@ -24,7 +27,6 @@ type BaseAwsResource struct {
2427
Nukables map[string]error
2528
Timeout time.Duration
2629
Context context.Context
27-
cancel context.CancelFunc
2830
}
2931

3032
func (br *BaseAwsResource) Init(cfg aws.Config) {
@@ -62,8 +64,10 @@ func (br *BaseAwsResource) GetAndSetResourceConfig(_ config.Config) config.Resou
6264
}
6365

6466
func (br *BaseAwsResource) PrepareContext(parentContext context.Context, resourceConfig config.ResourceType) error {
67+
br.Timeout = DefaultWaitTimeout
68+
br.Context = parentContext
69+
6570
if resourceConfig.Timeout == "" {
66-
br.Context = parentContext
6771
return nil
6872
}
6973

@@ -72,7 +76,14 @@ func (br *BaseAwsResource) PrepareContext(parentContext context.Context, resourc
7276
return err
7377
}
7478

75-
br.Context, _ = context.WithTimeout(parentContext, duration)
79+
br.Timeout = duration
80+
ctx, cancel := context.WithTimeout(parentContext, duration)
81+
br.Context = ctx
82+
// Note: cancel is intentionally not stored or deferred here.
83+
// The context will be cancelled when the parent context is cancelled,
84+
// or when the timeout expires. The parent context is managed by the
85+
// caller and will be cancelled when the nuke operation completes.
86+
_ = cancel
7687
return nil
7788
}
7889

@@ -107,3 +118,4 @@ func (br *BaseAwsResource) IsNukable(identifier string) (bool, error) {
107118

108119
return true, nil
109120
}
121+

‎aws/resources/ebs.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package resources
33
import (
44
"context"
55
goerr "errors"
6-
"time"
76

87
"github.com/aws/aws-sdk-go-v2/aws"
98
"github.com/aws/aws-sdk-go-v2/service/ec2"
@@ -118,7 +117,7 @@ func (ev *EBSVolumes) nukeAll(volumeIds []*string) error {
118117
waiter := ec2.NewVolumeDeletedWaiter(ev.Client)
119118
err := waiter.Wait(ev.Context, &ec2.DescribeVolumesInput{
120119
VolumeIds: aws.ToStringSlice(deletedVolumeIDs),
121-
}, 5*time.Minute)
120+
}, ev.Timeout)
122121
if err != nil {
123122
logging.Debugf("[Failed] %s", err)
124123
return errors.WithStackTrace(err)

‎aws/resources/ec2.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package resources
22

33
import (
44
"context"
5-
"time"
65

76
"github.com/aws/aws-sdk-go-v2/aws"
87
"github.com/aws/aws-sdk-go-v2/service/ec2"
@@ -161,7 +160,7 @@ func (ei *EC2Instances) nukeAll(instanceIds []*string) error {
161160
Values: aws.ToStringSlice(instanceIds),
162161
},
163162
},
164-
}, 15*time.Minute)
163+
}, ei.Timeout)
165164

166165
if err != nil {
167166
logging.Debugf("[Failed] %s", err)

‎aws/resources/ec2_network_interface.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ func (ni *NetworkInterface) nukeInstance(id *string) error {
212212
waiter := ec2.NewInstanceTerminatedWaiter(ni.Client)
213213
err = waiter.Wait(ni.Context, &ec2.DescribeInstancesInput{
214214
InstanceIds: []string{instanceID},
215-
}, 5*time.Minute)
215+
}, ni.Timeout)
216216
if err != nil {
217217
logging.Debugf("[nukeInstance] Instance termination waiting failed for instance %s: %v", instanceID, err)
218218
return errors.WithStackTrace(err)

‎aws/resources/ec2_vpc.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ func (v *EC2VPCs) nukeAll(vpcIds []string) error {
8888
}
8989

9090
var err error
91-
err = nuke(v.Client, v.ELBClient, id)
91+
err = nuke(v.Client, v.ELBClient, id, v.Timeout)
9292

9393
// Record status of this resource
9494
e := report.Entry{
@@ -111,7 +111,7 @@ func (v *EC2VPCs) nukeAll(vpcIds []string) error {
111111
return nil
112112
}
113113

114-
func nuke(client EC2VPCAPI, elbClient ELBClientAPI, vpcID string) error {
114+
func nuke(client EC2VPCAPI, elbClient ELBClientAPI, vpcID string, waitTimeout time.Duration) error {
115115
var err error
116116
// Note: order is quite important, otherwise you will encounter dependency violation errors.
117117

@@ -127,7 +127,7 @@ func nuke(client EC2VPCAPI, elbClient ELBClientAPI, vpcID string) error {
127127
return err
128128
}
129129

130-
err = nukeEc2Instances(client, vpcID)
130+
err = nukeEc2Instances(client, vpcID, waitTimeout)
131131
if err != nil {
132132
logging.Debug(fmt.Sprintf("Error nuking instances for VPC %s: %s", vpcID, err.Error()))
133133
return err
@@ -894,7 +894,7 @@ func nukeTargetGroups(client ELBClientAPI, vpcID string) error {
894894
return nil
895895
}
896896

897-
func nukeEc2Instances(client EC2VPCAPI, vpcID string) error {
897+
func nukeEc2Instances(client EC2VPCAPI, vpcID string, waitTimeout time.Duration) error {
898898
logging.Debug(fmt.Sprintf("Describing instances for %s", vpcID))
899899
output, err := client.DescribeInstances(context.Background(), &ec2.DescribeInstancesInput{
900900
Filters: []types.Filter{
@@ -933,12 +933,12 @@ func nukeEc2Instances(client EC2VPCAPI, vpcID string) error {
933933
return errors.WithStackTrace(err)
934934
}
935935

936-
// weight for terminate the instances
936+
// wait for terminate the instances
937937
logging.Debug(fmt.Sprintf("waiting for the instance to be terminated for %s", vpcID))
938938
waiter := ec2.NewInstanceTerminatedWaiter(client)
939939
err = waiter.Wait(context.Background(), &ec2.DescribeInstancesInput{
940940
InstanceIds: terminateInstancesIds,
941-
}, 5*time.Minute)
941+
}, waitTimeout)
942942
if err != nil {
943943
logging.Debug(fmt.Sprintf("Failed to wait instance termination for %s", vpcID))
944944
return errors.WithStackTrace(err)

‎aws/resources/ecs_service.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package resources
22

33
import (
44
"context"
5-
"time"
65

76
"github.com/aws/aws-sdk-go-v2/aws"
87
"github.com/aws/aws-sdk-go-v2/service/ecs"
@@ -160,7 +159,7 @@ func (services *ECSServices) waitUntilServicesDrained(ecsServiceArns []*string)
160159
}
161160

162161
waiter := ecs.NewServicesStableWaiter(services.Client)
163-
err := waiter.Wait(services.Context, params, 15*time.Minute)
162+
err := waiter.Wait(services.Context, params, services.Timeout)
164163
if err != nil {
165164
logging.Debugf("[Failed] Failed waiting for service to be stable %s: %s", *ecsServiceArn, err)
166165
} else {
@@ -202,7 +201,7 @@ func (services *ECSServices) waitUntilServicesDeleted(ecsServiceArns []*string)
202201
}
203202

204203
waiter := ecs.NewServicesInactiveWaiter(services.Client)
205-
err := waiter.Wait(services.Context, params, 15*time.Minute)
204+
err := waiter.Wait(services.Context, params, services.Timeout)
206205

207206
// Record status of this resource
208207
e := report.Entry{

‎aws/resources/eks.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package resources
33
import (
44
"context"
55
"sync"
6-
"time"
76

87
"github.com/aws/aws-sdk-go-v2/aws"
98
"github.com/aws/aws-sdk-go-v2/service/eks"
@@ -82,7 +81,7 @@ func (clusters *EKSClusters) deleteAsync(wg *sync.WaitGroup, errChan chan error,
8281
err := waiter.Wait(clusters.Context, &eks.DescribeNodegroupInput{
8382
ClusterName: aws.String(eksClusterName),
8483
NodegroupName: nodeGroup,
85-
}, 15*time.Minute)
84+
}, clusters.Timeout)
8685
if err != nil {
8786
logging.Debugf("[Failed] Failed waiting for Node Group %s associated with cluster %s to be deleted: %s", aws.ToString(nodeGroup), eksClusterName, err)
8887
allSubResourceErrs = multierror.Append(allSubResourceErrs, err)
@@ -185,7 +184,7 @@ func (clusters *EKSClusters) deleteEKSClusterFargateProfiles(eksClusterName stri
185184
waitErr := waiter.Wait(clusters.Context, &eks.DescribeFargateProfileInput{
186185
ClusterName: aws.String(eksClusterName),
187186
FargateProfileName: fargateProfile,
188-
}, 15*time.Minute)
187+
}, clusters.Timeout)
189188
if waitErr != nil {
190189
logging.Debugf("[Failed] Failed waiting for Fargate Profile %s associated with cluster %s to be deleted: %s", aws.ToString(fargateProfile), eksClusterName, waitErr)
191190
allDeleteErrs = multierror.Append(allDeleteErrs, waitErr)
@@ -206,7 +205,7 @@ func (clusters *EKSClusters) waitUntilEksClustersDeleted(eksClusterNames []*stri
206205
waiter := eks.NewClusterDeletedWaiter(clusters.Client)
207206
err := waiter.Wait(clusters.Context, &eks.DescribeClusterInput{
208207
Name: eksClusterName,
209-
}, 15*time.Minute)
208+
}, clusters.Timeout)
210209

211210
// Record status of this resource
212211
e := report.Entry{

‎aws/resources/elasticache.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"errors"
66
"fmt"
77
"strings"
8-
"time"
98

109
"github.com/aws/aws-sdk-go-v2/aws"
1110
"github.com/aws/aws-sdk-go-v2/service/elasticache"
@@ -126,7 +125,7 @@ func (cache *Elasticaches) nukeNonReplicationGroupElasticacheCluster(clusterId *
126125
&elasticache.DescribeCacheClustersInput{
127126
CacheClusterId: clusterId,
128127
},
129-
15*time.Minute,
128+
cache.Timeout,
130129
)
131130
}
132131

@@ -145,7 +144,7 @@ func (cache *Elasticaches) nukeReplicationGroupMemberElasticacheCluster(clusterI
145144
waitErr := waiter.Wait(
146145
cache.Context,
147146
&elasticache.DescribeReplicationGroupsInput{ReplicationGroupId: clusterId},
148-
15*time.Minute,
147+
cache.Timeout,
149148
)
150149

151150
if waitErr != nil {

‎aws/resources/elbv2.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package resources
22

33
import (
44
"context"
5-
"time"
65

76
"github.com/aws/aws-sdk-go-v2/aws"
87
"github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2"
@@ -69,7 +68,7 @@ func (balancer *LoadBalancersV2) nukeAll(arns []*string) error {
6968
waiter := elasticloadbalancingv2.NewLoadBalancersDeletedWaiter(balancer.Client)
7069
err := waiter.Wait(balancer.Context, &elasticloadbalancingv2.DescribeLoadBalancersInput{
7170
LoadBalancerArns: aws.ToStringSlice(deletedArns),
72-
}, 15*time.Minute)
71+
}, balancer.Timeout)
7372
if err != nil {
7473
logging.Debugf("[Failed] %s", err)
7574
return errors.WithStackTrace(err)

0 commit comments

Comments
 (0)