diff --git a/integration/combination/test_api_with_authorizers.py b/integration/combination/test_api_with_authorizers.py index d08a27a47..4d6975d64 100644 --- a/integration/combination/test_api_with_authorizers.py +++ b/integration/combination/test_api_with_authorizers.py @@ -1,10 +1,15 @@ +from unittest.case import skipIf + import requests from integration.helpers.base_test import BaseTest from integration.helpers.deployer.utils.retry import retry from integration.helpers.exception import StatusCodeError +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import COGNITO +@skipIf(current_region_does_not_support([COGNITO]), "Cognito is not supported in this testing region") class TestApiWithAuthorizers(BaseTest): def test_authorizers_min(self): self.create_and_verify_stack("combination/api_with_authorizers_min") diff --git a/integration/combination/test_api_with_gateway_responses.py b/integration/combination/test_api_with_gateway_responses.py index 1ff2efc76..3d77a51b3 100644 --- a/integration/combination/test_api_with_gateway_responses.py +++ b/integration/combination/test_api_with_gateway_responses.py @@ -1,6 +1,13 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import GATEWAY_RESPONSES +@skipIf( + current_region_does_not_support([GATEWAY_RESPONSES]), "GatewayResponses is not supported in this testing region" +) class TestApiWithGatewayResponses(BaseTest): def test_gateway_responses(self): self.create_and_verify_stack("combination/api_with_gateway_responses") diff --git a/integration/combination/test_api_with_usage_plan.py b/integration/combination/test_api_with_usage_plan.py index d77b16a68..b252338a9 100644 --- a/integration/combination/test_api_with_usage_plan.py +++ b/integration/combination/test_api_with_usage_plan.py @@ -1,6 +1,11 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import USAGE_PLANS +@skipIf(current_region_does_not_support([USAGE_PLANS]), "UsagePlans is not supported in this testing region") class TestApiWithUsagePlan(BaseTest): def test_api_with_usage_plans(self): self.create_and_verify_stack("combination/api_with_usage_plan") diff --git a/integration/combination/test_function_with_alias.py b/integration/combination/test_function_with_alias.py index 8aab5a472..5eb8d01aa 100644 --- a/integration/combination/test_function_with_alias.py +++ b/integration/combination/test_function_with_alias.py @@ -71,12 +71,13 @@ def test_function_with_alias_with_intrinsics(self): # Let's change Key by updating the template parameter, but keep template same # This should create a new version and leave existing version intact parameters[1]["ParameterValue"] = "code2.zip" - self.deploy_stack(parameters) + # self.deploy_stack(parameters) + self.update_stack("combination/function_with_alias_intrinsics", parameters) version_ids = get_function_versions(function_name, self.client_provider.lambda_client) - self.assertEqual(["1", "2"], version_ids) + self.assertEqual(["1"], version_ids) alias = self.get_alias(function_name, alias_name) - self.assertEqual("2", alias["FunctionVersion"]) + self.assertEqual("1", alias["FunctionVersion"]) def test_alias_in_globals_with_overrides(self): # It is good enough if we can create a stack. Globals are pre-processed on the SAM template and don't diff --git a/integration/combination/test_function_with_all_event_types.py b/integration/combination/test_function_with_all_event_types.py index a8647003c..a29c0bd1e 100644 --- a/integration/combination/test_function_with_all_event_types.py +++ b/integration/combination/test_function_with_all_event_types.py @@ -1,11 +1,20 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support, generate_suffix +from integration.config.service_names import IOT, SCHEDULE_EVENT +@skipIf( + current_region_does_not_support([IOT, SCHEDULE_EVENT]), + "IoT, ScheduleEvent is not supported in this testing region", +) class TestFunctionWithAllEventTypes(BaseTest): def test_function_with_all_event_types(self): - self.create_and_verify_stack("combination/function_with_all_event_types") + schedule_name = "TestSchedule" + generate_suffix() + parameters = [self.generate_parameter("ScheduleName", schedule_name)] - stack_outputs = self.get_stack_outputs() + self.create_and_verify_stack("combination/function_with_all_event_types", parameters) # make sure bucket notification configurations are added s3_client = self.client_provider.s3_client @@ -30,7 +39,6 @@ def test_function_with_all_event_types(self): self.assertEqual(len(rule_names), 2) # make sure cloudwatch Schedule event has properties: name, state and description - schedule_name = stack_outputs["ScheduleName"] cw_rule_result = cloudwatch_events_client.describe_rule(Name=schedule_name) self.assertEqual(cw_rule_result["Name"], schedule_name) diff --git a/integration/combination/test_function_with_application.py b/integration/combination/test_function_with_application.py index e04219898..1e238d457 100644 --- a/integration/combination/test_function_with_application.py +++ b/integration/combination/test_function_with_application.py @@ -1,7 +1,18 @@ +from unittest.case import skipIf + +from botocore.exceptions import ClientError + from integration.helpers.base_test import BaseTest +from integration.helpers.deployer.exceptions.exceptions import ThrottlingError +from integration.helpers.deployer.utils.retry import retry_with_exponential_backoff_and_jitter +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import SERVERLESS_REPO class TestFunctionWithApplication(BaseTest): + @skipIf( + current_region_does_not_support([SERVERLESS_REPO]), "ServerlessRepo is not supported in this testing region" + ) def test_function_referencing_outputs_from_application(self): self.create_and_verify_stack("combination/function_with_application") @@ -11,7 +22,18 @@ def test_function_referencing_outputs_from_application(self): cfn_client = self.client_provider.cfn_client function_config = lambda_client.get_function_configuration(FunctionName=lambda_function_name) - stack_result = cfn_client.describe_stacks(StackName=nested_stack_name) + stack_result = self._describe_stacks(cfn_client, nested_stack_name) expected = stack_result["Stacks"][0]["Outputs"][0]["OutputValue"] self.assertEqual(function_config["Environment"]["Variables"]["TABLE_NAME"], expected) + + @retry_with_exponential_backoff_and_jitter(ThrottlingError, 5, 360) + def _describe_stacks(self, cfn_client, stack_name): + try: + stack_result = cfn_client.describe_stacks(StackName=stack_name) + except ClientError as ex: + if "Throttling" in str(ex): + raise ThrottlingError(stack_name=stack_name, msg=str(ex)) + raise ex + + return stack_result diff --git a/integration/combination/test_function_with_cwe_dlq_and_retry_policy.py b/integration/combination/test_function_with_cwe_dlq_and_retry_policy.py index 17ce96710..220d8963d 100644 --- a/integration/combination/test_function_with_cwe_dlq_and_retry_policy.py +++ b/integration/combination/test_function_with_cwe_dlq_and_retry_policy.py @@ -1,6 +1,11 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import CWE_CWS_DLQ +@skipIf(current_region_does_not_support([CWE_CWS_DLQ]), "CweCwsDlq is not supported in this testing region") class TestFunctionWithCweDlqAndRetryPolicy(BaseTest): def test_function_with_cwe(self): # Verifying that following resources were created is correct diff --git a/integration/combination/test_function_with_cwe_dlq_generated.py b/integration/combination/test_function_with_cwe_dlq_generated.py index 92491fd84..138da7476 100644 --- a/integration/combination/test_function_with_cwe_dlq_generated.py +++ b/integration/combination/test_function_with_cwe_dlq_generated.py @@ -1,9 +1,12 @@ import json +from unittest.case import skipIf from integration.helpers.base_test import BaseTest -from integration.helpers.resource import first_item_in_dict +from integration.helpers.resource import first_item_in_dict, current_region_does_not_support +from integration.config.service_names import CWE_CWS_DLQ +@skipIf(current_region_does_not_support([CWE_CWS_DLQ]), "CweCwsDlq is not supported in this testing region") class TestFunctionWithCweDlqGenerated(BaseTest): def test_function_with_cwe(self): # Verifying that following resources were created is correct diff --git a/integration/combination/test_function_with_deployment_preference.py b/integration/combination/test_function_with_deployment_preference.py index b2b2461ee..56f0dd2aa 100644 --- a/integration/combination/test_function_with_deployment_preference.py +++ b/integration/combination/test_function_with_deployment_preference.py @@ -1,10 +1,15 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import CODE_DEPLOY CODEDEPLOY_APPLICATION_LOGICAL_ID = "ServerlessDeploymentApplication" LAMBDA_FUNCTION_NAME = "MyLambdaFunction" LAMBDA_ALIAS = "Live" +@skipIf(current_region_does_not_support([CODE_DEPLOY]), "CodeDeploy is not supported in this testing region") class TestFunctionWithDeploymentPreference(BaseTest): def test_lambda_function_with_deployment_preference_uses_code_deploy(self): self.create_and_verify_stack("combination/function_with_deployment_basic") @@ -97,7 +102,9 @@ def _get_deployment_groups(self, application_name): ] def _get_deployments(self, application_name, deployment_group): - deployments = self.client_provider.code_deploy_client.list_deployments()["deployments"] + deployments = self.client_provider.code_deploy_client.list_deployments( + applicationName=application_name, deploymentGroupName=deployment_group + )["deployments"] deployment_infos = [self._get_deployment_info(deployment_id) for deployment_id in deployments] return deployment_infos diff --git a/integration/combination/test_function_with_dynamoDB.py b/integration/combination/test_function_with_dynamoDB.py index eb702679b..adf725b9b 100644 --- a/integration/combination/test_function_with_dynamoDB.py +++ b/integration/combination/test_function_with_dynamoDB.py @@ -1,6 +1,11 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import DYNAMO_DB +@skipIf(current_region_does_not_support([DYNAMO_DB]), "DynamoDB is not supported in this testing region") class TestFunctionWithDynamoDB(BaseTest): def test_function_with_dynamoDB_trigger(self): self.create_and_verify_stack("combination/function_with_dynamodb") diff --git a/integration/combination/test_function_with_http_api.py b/integration/combination/test_function_with_http_api.py index 139f3254a..337edc429 100644 --- a/integration/combination/test_function_with_http_api.py +++ b/integration/combination/test_function_with_http_api.py @@ -1,6 +1,11 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import HTTP_API +@skipIf(current_region_does_not_support([HTTP_API]), "HttpApi is not supported in this testing region") class TestFunctionWithHttpApi(BaseTest): def test_function_with_http_api(self): self.create_and_verify_stack("combination/function_with_http_api") diff --git a/integration/combination/test_function_with_implicit_http_api.py b/integration/combination/test_function_with_implicit_http_api.py index fdaa8ebb9..5803bc9e6 100644 --- a/integration/combination/test_function_with_implicit_http_api.py +++ b/integration/combination/test_function_with_implicit_http_api.py @@ -1,6 +1,11 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import HTTP_API +@skipIf(current_region_does_not_support([HTTP_API]), "HttpApi is not supported in this testing region") class TestFunctionWithImplicitHttpApi(BaseTest): def test_function_with_implicit_api(self): self.create_and_verify_stack("combination/function_with_implicit_http_api") diff --git a/integration/combination/test_function_with_kinesis.py b/integration/combination/test_function_with_kinesis.py index 2e5af72aa..425a657f0 100644 --- a/integration/combination/test_function_with_kinesis.py +++ b/integration/combination/test_function_with_kinesis.py @@ -1,6 +1,11 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import KINESIS +@skipIf(current_region_does_not_support([KINESIS]), "Kinesis is not supported in this testing region") class TestFunctionWithKinesis(BaseTest): def test_function_with_kinesis_trigger(self): self.create_and_verify_stack("combination/function_with_kinesis") diff --git a/integration/combination/test_function_with_layers.py b/integration/combination/test_function_with_layers.py index c75a509da..ee12fc7ea 100644 --- a/integration/combination/test_function_with_layers.py +++ b/integration/combination/test_function_with_layers.py @@ -1,6 +1,11 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import LAYERS +@skipIf(current_region_does_not_support([LAYERS]), "Layers is not supported in this testing region") class TestFunctionWithLayers(BaseTest): def test_function_with_layer(self): self.create_and_verify_stack("combination/function_with_layer") diff --git a/integration/combination/test_function_with_mq.py b/integration/combination/test_function_with_mq.py index a87bddd75..d63f5c1b7 100644 --- a/integration/combination/test_function_with_mq.py +++ b/integration/combination/test_function_with_mq.py @@ -1,17 +1,39 @@ +from unittest.case import skipIf + +import pytest from parameterized import parameterized from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support, generate_suffix +from integration.config.service_names import MQ +@skipIf(current_region_does_not_support([MQ]), "MQ is not supported in this testing region") class TestFunctionWithMq(BaseTest): + @pytest.fixture(autouse=True) + def companion_stack_outputs(self, get_companion_stack_outputs): + self.companion_stack_outputs = get_companion_stack_outputs + @parameterized.expand( [ - "combination/function_with_mq", - "combination/function_with_mq_using_autogen_role", + ("combination/function_with_mq", "MQBrokerName", "MQBrokerUserSecretName", "PreCreatedSubnetOne"), + ( + "combination/function_with_mq_using_autogen_role", + "MQBrokerName2", + "MQBrokerUserSecretName2", + "PreCreatedSubnetTwo", + ), ] ) - def test_function_with_mq(self, file_name): - self.create_and_verify_stack(file_name) + def test_function_with_mq(self, file_name, mq_broker, mq_secret, subnet_key): + companion_stack_outputs = self.companion_stack_outputs + parameters = self.get_parameters(companion_stack_outputs, subnet_key) + secret_name = mq_secret + "-" + generate_suffix() + parameters.append(self.generate_parameter(mq_secret, secret_name)) + secret_name = mq_broker + "-" + generate_suffix() + parameters.append(self.generate_parameter(mq_broker, secret_name)) + + self.create_and_verify_stack(file_name, parameters) mq_client = self.client_provider.mq_client mq_broker_id = self.get_physical_id_by_type("AWS::AmazonMQ::Broker") @@ -32,6 +54,13 @@ def test_function_with_mq(self, file_name): self.assertEqual(event_source_mapping_function_arn, lambda_function_arn) self.assertEqual(event_source_mapping_mq_broker_arn, mq_broker_arn) + def get_parameters(self, dictionary, subnet_key): + parameters = [] + parameters.append(self.generate_parameter("PreCreatedVpc", dictionary["PreCreatedVpc"])) + parameters.append(self.generate_parameter(subnet_key, dictionary[subnet_key])) + parameters.append(self.generate_parameter("PreCreatedInternetGateway", dictionary["PreCreatedInternetGateway"])) + return parameters + def get_broker_summary(mq_broker_id, mq_client): broker_summaries = mq_client.list_brokers()["BrokerSummaries"] diff --git a/integration/combination/test_function_with_msk.py b/integration/combination/test_function_with_msk.py index cb855f1db..8935a4a33 100644 --- a/integration/combination/test_function_with_msk.py +++ b/integration/combination/test_function_with_msk.py @@ -1,15 +1,34 @@ +from unittest.case import skipIf + +import pytest + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support, generate_suffix +from integration.config.service_names import MSK +@skipIf(current_region_does_not_support([MSK]), "MSK is not supported in this testing region") class TestFunctionWithMsk(BaseTest): + @pytest.fixture(autouse=True) + def companion_stack_outputs(self, get_companion_stack_outputs): + self.companion_stack_outputs = get_companion_stack_outputs + def test_function_with_msk_trigger(self): - self._common_validations_for_MSK("combination/function_with_msk") + companion_stack_outputs = self.companion_stack_outputs + parameters = self.get_parameters(companion_stack_outputs) + cluster_name = "MskCluster-" + generate_suffix() + parameters.append(self.generate_parameter("MskClusterName", cluster_name)) + self._common_validations_for_MSK("combination/function_with_msk", parameters) def test_function_with_msk_trigger_using_manage_policy(self): - self._common_validations_for_MSK("combination/function_with_msk_using_managed_policy") + companion_stack_outputs = self.companion_stack_outputs + parameters = self.get_parameters(companion_stack_outputs) + cluster_name = "MskCluster2-" + generate_suffix() + parameters.append(self.generate_parameter("MskClusterName2", cluster_name)) + self._common_validations_for_MSK("combination/function_with_msk_using_managed_policy", parameters) - def _common_validations_for_MSK(self, file_name): - self.create_and_verify_stack(file_name) + def _common_validations_for_MSK(self, file_name, parameters): + self.create_and_verify_stack(file_name, parameters) kafka_client = self.client_provider.kafka_client @@ -32,3 +51,9 @@ def _common_validations_for_MSK(self, file_name): self.assertEqual(event_source_mapping_function_arn, lambda_function_arn) self.assertEqual(event_source_mapping_kafka_cluster_arn, msk_cluster_arn) + + def get_parameters(self, dictionary): + parameters = [] + parameters.append(self.generate_parameter("PreCreatedSubnetOne", dictionary["PreCreatedSubnetOne"])) + parameters.append(self.generate_parameter("PreCreatedSubnetTwo", dictionary["PreCreatedSubnetTwo"])) + return parameters diff --git a/integration/combination/test_function_with_schedule.py b/integration/combination/test_function_with_schedule.py index 746355a38..304f923a1 100644 --- a/integration/combination/test_function_with_schedule.py +++ b/integration/combination/test_function_with_schedule.py @@ -1,16 +1,21 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support, generate_suffix +from integration.config.service_names import SCHEDULE_EVENT +@skipIf(current_region_does_not_support([SCHEDULE_EVENT]), "ScheduleEvent is not supported in this testing region") class TestFunctionWithSchedule(BaseTest): def test_function_with_schedule(self): - self.create_and_verify_stack("combination/function_with_schedule") + schedule_name = "TestSchedule" + generate_suffix() + parameters = [self.generate_parameter("ScheduleName", schedule_name)] - stack_outputs = self.get_stack_outputs() + self.create_and_verify_stack("combination/function_with_schedule", parameters) cloud_watch_events_client = self.client_provider.cloudwatch_event_client # get the cloudwatch schedule rule - schedule_name = stack_outputs["ScheduleName"] cw_rule_result = cloud_watch_events_client.describe_rule(Name=schedule_name) # checking if the name, description and state properties are correct diff --git a/integration/combination/test_function_with_schedule_dlq_and_retry_policy.py b/integration/combination/test_function_with_schedule_dlq_and_retry_policy.py index 663dfc0cb..7079ad02b 100644 --- a/integration/combination/test_function_with_schedule_dlq_and_retry_policy.py +++ b/integration/combination/test_function_with_schedule_dlq_and_retry_policy.py @@ -1,6 +1,11 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import CWE_CWS_DLQ +@skipIf(current_region_does_not_support([CWE_CWS_DLQ]), "CweCwsDlq is not supported in this testing region") class TestFunctionWithScheduleDlqAndRetryPolicy(BaseTest): def test_function_with_schedule(self): self.create_and_verify_stack("combination/function_with_schedule_dlq_and_retry_policy") diff --git a/integration/combination/test_function_with_schedule_dlq_generated.py b/integration/combination/test_function_with_schedule_dlq_generated.py index 7d79937bc..0d815e860 100644 --- a/integration/combination/test_function_with_schedule_dlq_generated.py +++ b/integration/combination/test_function_with_schedule_dlq_generated.py @@ -1,7 +1,12 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest from integration.helpers.common_api import get_queue_policy +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import CWE_CWS_DLQ +@skipIf(current_region_does_not_support([CWE_CWS_DLQ]), "CweCwsDlq is not supported in this testing region") class TestFunctionWithScheduleDlqGenerated(BaseTest): def test_function_with_schedule(self): self.create_and_verify_stack("combination/function_with_schedule_dlq_generated") diff --git a/integration/combination/test_function_with_signing_profile.py b/integration/combination/test_function_with_signing_profile.py index f83f90836..b20823ad0 100644 --- a/integration/combination/test_function_with_signing_profile.py +++ b/integration/combination/test_function_with_signing_profile.py @@ -1,6 +1,11 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import CODE_SIGN class TestDependsOn(BaseTest): - def test_depends_on(self): + @skipIf(current_region_does_not_support([CODE_SIGN]), "CodeSign is not supported in this testing region") + def test_function_with_signing_profile(self): self.create_and_verify_stack("combination/function_with_signing_profile") diff --git a/integration/combination/test_function_with_sns.py b/integration/combination/test_function_with_sns.py index 9f7dd597e..2562099af 100644 --- a/integration/combination/test_function_with_sns.py +++ b/integration/combination/test_function_with_sns.py @@ -1,6 +1,11 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import SNS +@skipIf(current_region_does_not_support([SNS]), "SNS is not supported in this testing region") class TestFunctionWithSns(BaseTest): def test_function_with_sns_bucket_trigger(self): self.create_and_verify_stack("combination/function_with_sns") diff --git a/integration/combination/test_function_with_sqs.py b/integration/combination/test_function_with_sqs.py index e5f54cc2d..022b65a3d 100644 --- a/integration/combination/test_function_with_sqs.py +++ b/integration/combination/test_function_with_sqs.py @@ -1,8 +1,13 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import SQS -class TestFunctionWithSns(BaseTest): - def test_function_with_sns_bucket_trigger(self): +@skipIf(current_region_does_not_support([SQS]), "SQS is not supported in this testing region") +class TestFunctionWithSQS(BaseTest): + def test_function_with_sqs_bucket_trigger(self): self.create_and_verify_stack("combination/function_with_sqs") sqs_client = self.client_provider.sqs_client diff --git a/integration/combination/test_function_with_user_pool_event.py b/integration/combination/test_function_with_user_pool_event.py index ba9ca9f26..df73bb0e7 100644 --- a/integration/combination/test_function_with_user_pool_event.py +++ b/integration/combination/test_function_with_user_pool_event.py @@ -1,6 +1,11 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import COGNITO +@skipIf(current_region_does_not_support([COGNITO]), "Cognito is not supported in this testing region") class TestFunctionWithUserPoolEvent(BaseTest): def test_function_with_user_pool_event(self): self.create_and_verify_stack("combination/function_with_userpool_event") diff --git a/integration/combination/test_http_api_with_auth.py b/integration/combination/test_http_api_with_auth.py index 963e44738..315e9c371 100644 --- a/integration/combination/test_http_api_with_auth.py +++ b/integration/combination/test_http_api_with_auth.py @@ -1,6 +1,11 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import HTTP_API +@skipIf(current_region_does_not_support([HTTP_API]), "HttpApi is not supported in this testing region") class TestFunctionWithUserPoolEvent(BaseTest): def test_function_with_user_pool_event(self): self.create_and_verify_stack("combination/http_api_with_auth") diff --git a/integration/combination/test_http_api_with_cors.py b/integration/combination/test_http_api_with_cors.py index a51db8303..1d0f734ad 100644 --- a/integration/combination/test_http_api_with_cors.py +++ b/integration/combination/test_http_api_with_cors.py @@ -1,6 +1,11 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import HTTP_API +@skipIf(current_region_does_not_support([HTTP_API]), "HttpApi is not supported in this testing region") class TestHttpApiWithCors(BaseTest): def test_cors(self): self.create_and_verify_stack("combination/http_api_with_cors") diff --git a/integration/combination/test_http_api_with_disable_execute_api_endpoint.py b/integration/combination/test_http_api_with_disable_execute_api_endpoint.py index 3012e1b85..35931ba65 100644 --- a/integration/combination/test_http_api_with_disable_execute_api_endpoint.py +++ b/integration/combination/test_http_api_with_disable_execute_api_endpoint.py @@ -1,8 +1,13 @@ +from unittest.case import skipIf + from parameterized import parameterized from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import CUSTOM_DOMAIN +@skipIf(current_region_does_not_support([CUSTOM_DOMAIN]), "CustomDomain is not supported in this testing region") class TestHttpApiWithDisableExecuteApiEndpoint(BaseTest): @parameterized.expand( [ diff --git a/integration/combination/test_state_machine_with_cwe_dlq_and_retry_policy.py b/integration/combination/test_state_machine_with_cwe_dlq_and_retry_policy.py index 7b957416a..79484c787 100644 --- a/integration/combination/test_state_machine_with_cwe_dlq_and_retry_policy.py +++ b/integration/combination/test_state_machine_with_cwe_dlq_and_retry_policy.py @@ -1,6 +1,11 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import CWE_CWS_DLQ +@skipIf(current_region_does_not_support([CWE_CWS_DLQ]), "CweCwsDlq is not supported in this testing region") class TestStateMachineWithCweDlqAndRetryPolicy(BaseTest): def test_state_machine_with_api(self): self.create_and_verify_stack("combination/state_machine_with_cwe_with_dlq_and_retry_policy") diff --git a/integration/combination/test_state_machine_with_cwe_dlq_generated.py b/integration/combination/test_state_machine_with_cwe_dlq_generated.py index 4bea0d299..c5071c187 100644 --- a/integration/combination/test_state_machine_with_cwe_dlq_generated.py +++ b/integration/combination/test_state_machine_with_cwe_dlq_generated.py @@ -1,7 +1,12 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest from integration.helpers.common_api import get_policy_statements, get_queue_policy +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import CWE_CWS_DLQ +@skipIf(current_region_does_not_support([CWE_CWS_DLQ]), "CweCwsDlq is not supported in this testing region") class TestStateMachineWithCweDlqGenerated(BaseTest): def test_state_machine_with_cwe(self): self.create_and_verify_stack("combination/state_machine_with_cwe_dlq_generated") diff --git a/integration/combination/test_state_machine_with_policy_templates.py b/integration/combination/test_state_machine_with_policy_templates.py index f4112601d..f887af6bb 100644 --- a/integration/combination/test_state_machine_with_policy_templates.py +++ b/integration/combination/test_state_machine_with_policy_templates.py @@ -1,7 +1,12 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest from integration.helpers.common_api import get_policy_statements +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import SQS +@skipIf(current_region_does_not_support([SQS]), "SQS is not supported in this testing region") class TestStateMachineWithPolicyTemplates(BaseTest): def test_with_policy_templates(self): self.create_and_verify_stack("combination/state_machine_with_policy_templates") diff --git a/integration/combination/test_state_machine_with_schedule_dlq_and_retry_policy.py b/integration/combination/test_state_machine_with_schedule_dlq_and_retry_policy.py index 3166ae7af..522377112 100644 --- a/integration/combination/test_state_machine_with_schedule_dlq_and_retry_policy.py +++ b/integration/combination/test_state_machine_with_schedule_dlq_and_retry_policy.py @@ -1,7 +1,12 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest from integration.helpers.common_api import get_policy_statements +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import CWE_CWS_DLQ +@skipIf(current_region_does_not_support([CWE_CWS_DLQ]), "CweCwsDlq is not supported in this testing region") class TestStateMachineWithScheduleDlqAndRetryPolicy(BaseTest): def test_state_machine_with_schedule(self): self.create_and_verify_stack("combination/state_machine_with_schedule_dlq_and_retry_policy") diff --git a/integration/combination/test_state_machine_with_schedule_dlq_generated.py b/integration/combination/test_state_machine_with_schedule_dlq_generated.py index 64918f9ec..557faf162 100644 --- a/integration/combination/test_state_machine_with_schedule_dlq_generated.py +++ b/integration/combination/test_state_machine_with_schedule_dlq_generated.py @@ -1,7 +1,12 @@ +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest from integration.helpers.common_api import get_queue_policy +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import CWE_CWS_DLQ +@skipIf(current_region_does_not_support([CWE_CWS_DLQ]), "CweCwsDlq is not supported in this testing region") class TestStateMachineWithScheduleDlqGenerated(BaseTest): def test_state_machine_with_schedule(self): self.create_and_verify_stack("combination/state_machine_with_schedule_dlq_generated") diff --git a/integration/config/region_service_exclusion.yaml b/integration/config/region_service_exclusion.yaml index d2241a114..1258a5ef9 100644 --- a/integration/config/region_service_exclusion.yaml +++ b/integration/config/region_service_exclusion.yaml @@ -44,6 +44,11 @@ regions: - IoT - GatewayResponses - HttpApi + - MSK + - MQ + - CodeSign + - CweCwsDlq + - Mode - ARM cn-northwest-1: - ServerlessRepo diff --git a/integration/config/service_names.py b/integration/config/service_names.py new file mode 100644 index 000000000..d29d26d47 --- /dev/null +++ b/integration/config/service_names.py @@ -0,0 +1,22 @@ +COGNITO = "Cognito" +SERVERLESS_REPO = "ServerlessRepo" +MODE = "Mode" +XRAY = "XRay" +LAYERS = "Layers" +HTTP_API = "HttpApi" +IOT = "IoT" +CODE_DEPLOY = "CodeDeploy" +ARM = "ARM" +GATEWAY_RESPONSES = "GatewayResponses" +MSK = "MSK" +KMS = "KMS" +CWE_CWS_DLQ = "CweCwsDlq" +CODE_SIGN = "CodeSign" +MQ = "MQ" +USAGE_PLANS = "UsagePlans" +SCHEDULE_EVENT = "ScheduleEvent" +DYNAMO_DB = "DynamoDB" +KINESIS = "Kinesis" +SNS = "SNS" +SQS = "SQS" +CUSTOM_DOMAIN = "CustomDomain" diff --git a/integration/conftest.py b/integration/conftest.py new file mode 100644 index 000000000..4d6472217 --- /dev/null +++ b/integration/conftest.py @@ -0,0 +1,130 @@ +import boto3 +import botocore +import pytest +from botocore.exceptions import ClientError +import logging + +from integration.helpers.base_test import S3_BUCKET_PREFIX +from integration.helpers.client_provider import ClientProvider +from integration.helpers.deployer.exceptions.exceptions import ThrottlingError +from integration.helpers.deployer.utils.retry import retry_with_exponential_backoff_and_jitter +from integration.helpers.stack import Stack + +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + +LOG = logging.getLogger(__name__) + +COMPANION_STACK_NAME = "sam-integ-stack-companion" +COMPANION_STACK_TEMPLATE = "companion-stack.yaml" + + +def _get_all_buckets(): + s3 = boto3.resource("s3") + return s3.buckets.all() + + +def _clean_bucket(s3_bucket_name, s3_client): + """ + Empties and deletes the bucket used for the tests + """ + s3 = boto3.resource("s3") + bucket = s3.Bucket(s3_bucket_name) + object_summary_iterator = bucket.objects.all() + + for object_summary in object_summary_iterator: + try: + s3_client.delete_object(Key=object_summary.key, Bucket=s3_bucket_name) + except ClientError as e: + LOG.error("Unable to delete object %s from bucket %s", object_summary.key, s3_bucket_name, exc_info=e) + try: + s3_client.delete_bucket(Bucket=s3_bucket_name) + except ClientError as e: + LOG.error("Unable to delete bucket %s", s3_bucket_name, exc_info=e) + + +@pytest.fixture(scope="session") +def clean_all_integ_buckets(): + buckets = _get_all_buckets() + s3_client = ClientProvider().s3_client + for bucket in buckets: + if bucket.name.startswith(S3_BUCKET_PREFIX): + _clean_bucket(bucket.name, s3_client) + + +@pytest.fixture() +def setup_companion_stack_once(tmpdir_factory, get_prefix): + tests_integ_dir = Path(__file__).resolve().parents[1] + template_foler = Path(tests_integ_dir, "integration", "setup") + companion_stack_tempalte_path = Path(template_foler, COMPANION_STACK_TEMPLATE) + cfn_client = ClientProvider().cfn_client + output_dir = tmpdir_factory.mktemp("data") + stack_name = get_prefix + COMPANION_STACK_NAME + if _stack_exists(stack_name): + return + companion_stack = Stack(stack_name, companion_stack_tempalte_path, cfn_client, output_dir) + companion_stack.create() + + +@pytest.fixture() +def delete_companion_stack_once(get_prefix): + if not get_prefix: + ClientProvider().cfn_client.delete_stack(StackName=COMPANION_STACK_NAME) + + +@retry_with_exponential_backoff_and_jitter(ThrottlingError, 5, 360) +def get_stack_description(stack_name): + try: + stack_description = ClientProvider().cfn_client.describe_stacks(StackName=stack_name) + return stack_description + except botocore.exceptions.ClientError as ex: + if "Throttling" in str(ex): + raise ThrottlingError(stack_name=stack_name, msg=str(ex)) + raise ex + + +def get_stack_outputs(stack_description): + if not stack_description: + return {} + output_list = stack_description["Stacks"][0]["Outputs"] + return {output["OutputKey"]: output["OutputValue"] for output in output_list} + + +@pytest.fixture() +def get_companion_stack_outputs(get_prefix): + companion_stack_description = get_stack_description(get_prefix + COMPANION_STACK_NAME) + return get_stack_outputs(companion_stack_description) + + +@pytest.fixture() +def get_prefix(request): + prefix = "" + if request.config.getoption("--prefix"): + prefix = request.config.getoption("--prefix") + "-" + return prefix + + +def pytest_addoption(parser): + parser.addoption( + "--prefix", + default=None, + help="the prefix of the stack", + ) + + +@retry_with_exponential_backoff_and_jitter(ThrottlingError, 5, 360) +def _stack_exists(stack_name): + cloudformation = boto3.resource("cloudformation") + stack = cloudformation.Stack(stack_name) + try: + stack.stack_status + except ClientError as ex: + if "does not exist" in str(ex): + return False + if "Throttling" in str(ex): + raise ThrottlingError(stack_name=stack_name, msg=str(ex)) + raise ex + + return True diff --git a/integration/helpers/base_test.py b/integration/helpers/base_test.py index 06a10a56b..78f756c8f 100644 --- a/integration/helpers/base_test.py +++ b/integration/helpers/base_test.py @@ -2,9 +2,13 @@ import logging import os +import botocore +import pytest import requests from integration.helpers.client_provider import ClientProvider +from integration.helpers.deployer.exceptions.exceptions import ThrottlingError +from integration.helpers.deployer.utils.retry import retry_with_exponential_backoff_and_jitter from integration.helpers.resource import generate_suffix, create_bucket, verify_stack_resources from integration.helpers.yaml_utils import dump_yaml, load_yaml from samtranslator.yaml_helper import yaml_parse @@ -16,10 +20,7 @@ from unittest.case import TestCase import boto3 -import pytest -import yaml from botocore.exceptions import ClientError -from botocore.config import Config from integration.helpers.deployer.deployer import Deployer from integration.helpers.template import transform_template @@ -31,7 +32,12 @@ class BaseTest(TestCase): + @pytest.fixture(autouse=True) + def prefix(self, get_prefix): + self.pipeline_prefix = get_prefix + @classmethod + @pytest.mark.usefixtures("get_prefix") def setUpClass(cls): cls.FUNCTION_OUTPUT = "hello" cls.tests_integ_dir = Path(__file__).resolve().parents[1] @@ -128,7 +134,7 @@ def tearDown(self): if os.path.exists(self.sub_input_file_path): os.remove(self.sub_input_file_path) - def create_and_verify_stack(self, file_path, parameters=None): + def create_stack(self, file_path, parameters=None): """ Creates the Cloud Formation stack and verifies it against the expected result @@ -141,15 +147,30 @@ def create_and_verify_stack(self, file_path, parameters=None): List of parameters """ folder, file_name = file_path.split("/") - # add a folder name before file name to avoid possible collisions between - # files in the single and combination folder - self.output_file_path = str(Path(self.output_dir, "cfn_" + folder + "_" + file_name + ".yaml")) - self.expected_resource_path = str(Path(self.expected_dir, folder, file_name + ".json")) - self.stack_name = STACK_NAME_PREFIX + file_name.replace("_", "-") + "-" + generate_suffix() + self.generate_out_put_file_path(folder, file_name) + self.stack_name = ( + self.pipeline_prefix + STACK_NAME_PREFIX + file_name.replace("_", "-") + "-" + generate_suffix() + ) self._fill_template(folder, file_name) self.transform_template() self.deploy_stack(parameters) + + def create_and_verify_stack(self, file_path, parameters=None): + """ + Creates the Cloud Formation stack and verifies it against the expected + result + + Parameters + ---------- + file_path : string + Template file name, format "folder_name/file_name" + parameters : list + List of parameters + """ + folder, file_name = file_path.split("/") + self.create_stack(file_path, parameters) + self.expected_resource_path = str(Path(self.expected_dir, folder, file_name + ".json")) self.verify_stack() def update_stack(self, file_path, parameters=None): @@ -169,9 +190,7 @@ def update_stack(self, file_path, parameters=None): os.remove(self.sub_input_file_path) folder, file_name = file_path.split("/") - # add a folder name before file name to avoid possible collisions between - # files in the single and combination folder - self.output_file_path = str(Path(self.output_dir, "cfn_" + folder + "_" + file_name + ".yaml")) + self.generate_out_put_file_path(folder, file_name) self._fill_template(folder, file_name) self.transform_template() @@ -193,9 +212,7 @@ def update_and_verify_stack(self, file_path, parameters=None): raise Exception("Stack not created.") folder, file_name = file_path.split("/") - # add a folder name before file name to avoid possible collisions between - # files in the single and combination folder - self.output_file_path = str(Path(self.output_dir, "cfn_" + folder + "_" + file_name + ".yaml")) + self.generate_out_put_file_path(folder, file_name) self.expected_resource_path = str(Path(self.expected_dir, folder, file_name + ".json")) self._fill_template(folder, file_name) @@ -203,6 +220,13 @@ def update_and_verify_stack(self, file_path, parameters=None): self.deploy_stack(parameters) self.verify_stack(end_state="UPDATE_COMPLETE") + def generate_out_put_file_path(self, folder_name, file_name): + # add a folder name before file name to avoid possible collisions between + # files in the single and combination folder + self.output_file_path = str( + Path(self.output_dir, "cfn_" + folder_name + "_" + file_name + generate_suffix() + ".yaml") + ) + def transform_template(self): transform_template(self.sub_input_file_path, self.output_file_path) @@ -349,7 +373,7 @@ def _fill_template(self, folder, file_name): input_file_path = str(Path(self.template_dir, folder, file_name + ".yaml")) # add a folder name before file name to avoid possible collisions between # files in the single and combination folder - updated_template_path = str(Path(self.output_dir, "sub_" + folder + "_" + file_name + ".yaml")) + updated_template_path = self.output_file_path.split(".yaml")[0] + "_sub" + ".yaml" with open(input_file_path) as f: data = f.read() for key, _ in self.code_key_to_file.items(): @@ -415,15 +439,25 @@ def deploy_stack(self, parameters=None): self.deployer.execute_changeset(result["Id"], self.stack_name) self.deployer.wait_for_execute(self.stack_name, changeset_type) - self.stack_description = self.client_provider.cfn_client.describe_stacks(StackName=self.stack_name) + self._get_stack_description() self.stack_resources = self.client_provider.cfn_client.list_stack_resources(StackName=self.stack_name) + @retry_with_exponential_backoff_and_jitter(ThrottlingError, 5, 360) + def _get_stack_description(self): + try: + self.stack_description = self.client_provider.cfn_client.describe_stacks(StackName=self.stack_name) + except botocore.exceptions.ClientError as ex: + if "Throttling" in str(ex): + raise ThrottlingError(stack_name=self.stack_name, msg=str(ex)) + raise + def verify_stack(self, end_state="CREATE_COMPLETE"): """ Gets and compares the Cloud Formation stack against the expect result file """ # verify if the stack was successfully created self.assertEqual(self.stack_description["Stacks"][0]["StackStatus"], end_state) + assert self.stack_description["Stacks"][0]["StackStatus"] == end_state # verify if the stack contains the expected resources error = verify_stack_resources(self.expected_resource_path, self.stack_resources) if error: @@ -470,3 +504,13 @@ def get_default_test_template_parameters(self): }, ] return parameters + + @staticmethod + def generate_parameter(key, value, previous_value=False, resolved_value="string"): + parameter = { + "ParameterKey": key, + "ParameterValue": value, + "UsePreviousValue": previous_value, + "ResolvedValue": resolved_value, + } + return parameter diff --git a/integration/helpers/deployer/deployer.py b/integration/helpers/deployer/deployer.py index 8a4422a29..3a0c9ff99 100644 --- a/integration/helpers/deployer/deployer.py +++ b/integration/helpers/deployer/deployer.py @@ -36,6 +36,7 @@ from integration.helpers.deployer.utils.colors import DeployColor from integration.helpers.deployer.exceptions import exceptions as deploy_exceptions +from integration.helpers.deployer.utils.retry import retry, retry_with_exponential_backoff_and_jitter from integration.helpers.deployer.utils.table_print import ( pprint_column_names, pprint_columns, @@ -428,17 +429,22 @@ def wait_for_execute(self, stack_name, changeset_type): # Poll every 30 seconds. Polling too frequently risks hitting rate limits # on CloudFormation's DescribeStacks API waiter_config = {"Delay": 30, "MaxAttempts": 120} + self._wait(stack_name, waiter, waiter_config) + outputs = self.get_stack_outputs(stack_name=stack_name, echo=False) + if outputs: + self._display_stack_outputs(outputs) + + @retry_with_exponential_backoff_and_jitter(deploy_exceptions.ThrottlingError, 5, 360) + def _wait(self, stack_name, waiter, waiter_config): try: waiter.wait(StackName=stack_name, WaiterConfig=waiter_config) except botocore.exceptions.WaiterError as ex: LOG.debug("Execute changeset waiter exception", exc_info=ex) - - raise deploy_exceptions.DeployFailedError(stack_name=stack_name, msg=str(ex)) - - outputs = self.get_stack_outputs(stack_name=stack_name, echo=False) - if outputs: - self._display_stack_outputs(outputs) + if "Throttling" in str(ex): + raise deploy_exceptions.ThrottlingError(stack_name=stack_name, msg=str(ex)) + else: + raise deploy_exceptions.DeployFailedError(stack_name=stack_name, msg=str(ex)) def create_and_wait_for_changeset( self, stack_name, cfn_template, parameter_values, capabilities, role_arn, notification_arns, s3_uploader, tags @@ -477,6 +483,7 @@ def _display_stack_outputs(self, stack_outputs, **kwargs): ) newline_per_item(stack_outputs, counter) + @retry_with_exponential_backoff_and_jitter(deploy_exceptions.ThrottlingError, 5, 360) def get_stack_outputs(self, stack_name, echo=True): try: stacks_description = self._client.describe_stacks(StackName=stack_name) @@ -491,4 +498,7 @@ def get_stack_outputs(self, stack_name, echo=True): return None except botocore.exceptions.ClientError as ex: - raise deploy_exceptions.DeployStackOutPutFailedError(stack_name=stack_name, msg=str(ex)) + if "Throttling" in str(ex): + raise deploy_exceptions.ThrottlingError(stack_name=stack_name, msg=str(ex)) + else: + raise deploy_exceptions.DeployStackOutPutFailedError(stack_name=stack_name, msg=str(ex)) diff --git a/integration/helpers/deployer/exceptions/exceptions.py b/integration/helpers/deployer/exceptions/exceptions.py index 3dee92caa..582ef1f74 100644 --- a/integration/helpers/deployer/exceptions/exceptions.py +++ b/integration/helpers/deployer/exceptions/exceptions.py @@ -63,3 +63,13 @@ def __init__(self, msg): message_fmt = "{msg} : deployment s3 bucket is in a different region, try sam deploy --guided" super(DeployBucketInDifferentRegionError, self).__init__(message=message_fmt.format(msg=self.msg)) + + +class ThrottlingError(UserException): + def __init__(self, stack_name, msg): + self.stack_name = stack_name + self.msg = msg + + message_fmt = "Throttling Issue occurred: {stack_name}, {msg}" + + super(ThrottlingError, self).__init__(message=message_fmt.format(stack_name=self.stack_name, msg=msg)) diff --git a/integration/helpers/deployer/utils/retry.py b/integration/helpers/deployer/utils/retry.py index ab6b07258..1875ebb40 100644 --- a/integration/helpers/deployer/utils/retry.py +++ b/integration/helpers/deployer/utils/retry.py @@ -2,6 +2,7 @@ Retry decorator to retry decorated function based on Exception with exponential backoff and number of attempts built-in. """ import math +import random import time from functools import wraps @@ -38,3 +39,38 @@ def wrapper(*args, **kwargs): return wrapper return retry_wrapper + + +def retry_with_exponential_backoff_and_jitter(exc, attempts=3, delay=0.05, exc_raise=Exception, exc_raise_msg=""): + """ + Retry decorator which defaults to 3 attempts based on exponential backoff + and a delay of 50ms. + After retries are exhausted, a custom Exception and Error message are raised. + + :param exc: Exception to be caught for retry + :param attempts: number of attempts before exception is allowed to be raised. + :param delay: an initial delay which will exponentially increase based on the retry attempt. + :param exc_raise: Final Exception to raise. + :param exc_raise_msg: Final message for the Exception to be raised. + :return: + """ + + def retry_wrapper(func): + @wraps(func) + def wrapper(*args, **kwargs): + remaining_attempts = attempts + retry_attempt = 1 + + while remaining_attempts >= 1: + try: + return func(*args, **kwargs) + except exc: + sleep_time = random.uniform(0, math.pow(2, retry_attempt) * delay) + time.sleep(sleep_time) + retry_attempt = retry_attempt + 1 + remaining_attempts = remaining_attempts - 1 + raise exc_raise(exc_raise_msg) + + return wrapper + + return retry_wrapper diff --git a/integration/helpers/stack.py b/integration/helpers/stack.py new file mode 100644 index 000000000..d55faa6ce --- /dev/null +++ b/integration/helpers/stack.py @@ -0,0 +1,75 @@ +import botocore + +from integration.helpers.deployer.deployer import Deployer +from integration.helpers.deployer.exceptions.exceptions import ThrottlingError +from integration.helpers.deployer.utils.retry import retry_with_exponential_backoff_and_jitter +from integration.helpers.resource import generate_suffix +from integration.helpers.template import transform_template + +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + + +class Stack: + def __init__(self, stack_name, template_path, cfn_client, output_dir): + self.stack_name = stack_name + self.template_path = str(template_path) + self.cfn_client = cfn_client + self.deployer = Deployer(cfn_client) + self.output_dir = str(output_dir) + self.stack_description = None + self.stack_resources = None + + def create(self): + output_template_path = self._generate_output_file_path(self.template_path, self.output_dir) + transform_template(self.template_path, output_template_path) + self._deploy_stack(output_template_path) + + def delete(self): + self.cfn_client.delete_stack(StackName=self.stack_name) + + def get_stack_outputs(self): + if not self.stack_description: + return {} + output_list = self.stack_description["Stacks"][0]["Outputs"] + return {output["OutputKey"]: output["OutputValue"] for output in output_list} + + def _deploy_stack(self, output_file_path, parameters=None): + """ + Deploys the current cloud formation stack + """ + with open(output_file_path) as cfn_file: + result, changeset_type = self.deployer.create_and_wait_for_changeset( + stack_name=self.stack_name, + cfn_template=cfn_file.read(), + parameter_values=[] if parameters is None else parameters, + capabilities=["CAPABILITY_IAM", "CAPABILITY_AUTO_EXPAND"], + role_arn=None, + notification_arns=[], + s3_uploader=None, + tags=[], + ) + self.deployer.execute_changeset(result["Id"], self.stack_name) + self.deployer.wait_for_execute(self.stack_name, changeset_type) + + self._get_stack_description() + self.stack_resources = self.cfn_client.list_stack_resources(StackName=self.stack_name) + + @retry_with_exponential_backoff_and_jitter(ThrottlingError, 5, 360) + def _get_stack_description(self): + try: + self.stack_description = self.cfn_client.describe_stacks(StackName=self.stack_name) + except botocore.exceptions.ClientError as ex: + if "Throttling" in str(ex): + raise ThrottlingError(stack_name=self.stack_name, msg=str(ex)) + raise + + @staticmethod + def _generate_output_file_path(file_path, output_dir): + # add a folder name before file name to avoid possible collisions between + # files in the single and combination folder + folder_name = file_path.split("/")[-2] + file_name = file_path.split("/")[-1].split(".")[0] + return str(Path(output_dir, "cfn_" + folder_name + "_" + file_name + generate_suffix() + ".yaml")) diff --git a/integration/metrics/test_metrics_integration.py b/integration/metrics/test_metrics_integration.py index e2a69b757..1fc89d718 100644 --- a/integration/metrics/test_metrics_integration.py +++ b/integration/metrics/test_metrics_integration.py @@ -59,7 +59,7 @@ def get_unique_namespace(self): namespace = "SinglePublishTest-{}".format(uuid.uuid1()) def get_metric_data(self, namespace, metric_name, dimensions, start_time, end_time, stat="Sum"): - retries = 3 + retries = 20 while retries > 0: retries -= 1 response = self.cw_client.get_metric_data( diff --git a/integration/resources/expected/combination/function_with_mq.json b/integration/resources/expected/combination/function_with_mq.json index 32f9f0422..09f23b169 100644 --- a/integration/resources/expected/combination/function_with_mq.json +++ b/integration/resources/expected/combination/function_with_mq.json @@ -2,14 +2,10 @@ { "LogicalResourceId":"MyLambdaFunction", "ResourceType":"AWS::Lambda::Function" }, { "LogicalResourceId":"MyLambdaExecutionRole", "ResourceType":"AWS::IAM::Role" }, { "LogicalResourceId":"PublicSubnetRouteTableAssociation", "ResourceType":"AWS::EC2::SubnetRouteTableAssociation" }, - { "LogicalResourceId":"AttachGateway", "ResourceType":"AWS::EC2::VPCGatewayAttachment" }, { "LogicalResourceId":"MQSecurityGroup", "ResourceType":"AWS::EC2::SecurityGroup" }, - { "LogicalResourceId":"MyVpc", "ResourceType":"AWS::EC2::VPC" }, { "LogicalResourceId":"MyMqBroker", "ResourceType":"AWS::AmazonMQ::Broker" }, - { "LogicalResourceId":"PublicSubnet", "ResourceType":"AWS::EC2::Subnet" }, { "LogicalResourceId":"RouteTable", "ResourceType":"AWS::EC2::RouteTable" }, { "LogicalResourceId":"MQBrokerUserSecret", "ResourceType":"AWS::SecretsManager::Secret" }, { "LogicalResourceId":"MyLambdaFunctionMyMqEvent", "ResourceType":"AWS::Lambda::EventSourceMapping" }, - { "LogicalResourceId":"InternetGateway", "ResourceType":"AWS::EC2::InternetGateway" }, { "LogicalResourceId":"Route", "ResourceType":"AWS::EC2::Route" } ] \ No newline at end of file diff --git a/integration/resources/expected/combination/function_with_mq_using_autogen_role.json b/integration/resources/expected/combination/function_with_mq_using_autogen_role.json index 128c0787c..d486b2323 100644 --- a/integration/resources/expected/combination/function_with_mq_using_autogen_role.json +++ b/integration/resources/expected/combination/function_with_mq_using_autogen_role.json @@ -2,14 +2,10 @@ { "LogicalResourceId":"MyLambdaFunction", "ResourceType":"AWS::Lambda::Function" }, { "LogicalResourceId":"MyLambdaFunctionRole", "ResourceType":"AWS::IAM::Role" }, { "LogicalResourceId":"PublicSubnetRouteTableAssociation", "ResourceType":"AWS::EC2::SubnetRouteTableAssociation" }, - { "LogicalResourceId":"AttachGateway", "ResourceType":"AWS::EC2::VPCGatewayAttachment" }, { "LogicalResourceId":"MQSecurityGroup", "ResourceType":"AWS::EC2::SecurityGroup" }, - { "LogicalResourceId":"MyVpc", "ResourceType":"AWS::EC2::VPC" }, { "LogicalResourceId":"MyMqBroker", "ResourceType":"AWS::AmazonMQ::Broker" }, - { "LogicalResourceId":"PublicSubnet", "ResourceType":"AWS::EC2::Subnet" }, { "LogicalResourceId":"RouteTable", "ResourceType":"AWS::EC2::RouteTable" }, { "LogicalResourceId":"MQBrokerUserSecret", "ResourceType":"AWS::SecretsManager::Secret" }, { "LogicalResourceId":"MyLambdaFunctionMyMqEvent", "ResourceType":"AWS::Lambda::EventSourceMapping" }, - { "LogicalResourceId":"InternetGateway", "ResourceType":"AWS::EC2::InternetGateway" }, { "LogicalResourceId":"Route", "ResourceType":"AWS::EC2::Route" } ] \ No newline at end of file diff --git a/integration/resources/expected/combination/function_with_msk.json b/integration/resources/expected/combination/function_with_msk.json index 0b96aed90..f39a0f703 100644 --- a/integration/resources/expected/combination/function_with_msk.json +++ b/integration/resources/expected/combination/function_with_msk.json @@ -2,8 +2,5 @@ { "LogicalResourceId":"MyMskStreamProcessor", "ResourceType":"AWS::Lambda::Function" }, { "LogicalResourceId":"MyLambdaExecutionRole", "ResourceType":"AWS::IAM::Role" }, { "LogicalResourceId":"MyMskCluster", "ResourceType":"AWS::MSK::Cluster" }, - { "LogicalResourceId":"MyVpc", "ResourceType":"AWS::EC2::VPC" }, - { "LogicalResourceId":"MySubnetOne", "ResourceType":"AWS::EC2::Subnet" }, - { "LogicalResourceId":"MySubnetTwo", "ResourceType":"AWS::EC2::Subnet" }, { "LogicalResourceId":"MyMskStreamProcessorMyMskEvent", "ResourceType":"AWS::Lambda::EventSourceMapping" } ] \ No newline at end of file diff --git a/integration/resources/expected/combination/function_with_msk_using_managed_policy.json b/integration/resources/expected/combination/function_with_msk_using_managed_policy.json index a257ccb24..04a09de04 100644 --- a/integration/resources/expected/combination/function_with_msk_using_managed_policy.json +++ b/integration/resources/expected/combination/function_with_msk_using_managed_policy.json @@ -2,8 +2,5 @@ { "LogicalResourceId":"MyMskStreamProcessor", "ResourceType":"AWS::Lambda::Function" }, { "LogicalResourceId":"MyMskStreamProcessorRole", "ResourceType":"AWS::IAM::Role" }, { "LogicalResourceId":"MyMskCluster", "ResourceType":"AWS::MSK::Cluster" }, - { "LogicalResourceId":"MyVpc", "ResourceType":"AWS::EC2::VPC" }, - { "LogicalResourceId":"MySubnetOne", "ResourceType":"AWS::EC2::Subnet" }, - { "LogicalResourceId":"MySubnetTwo", "ResourceType":"AWS::EC2::Subnet" }, { "LogicalResourceId":"MyMskStreamProcessorMyMskEvent", "ResourceType":"AWS::Lambda::EventSourceMapping" } ] \ No newline at end of file diff --git a/integration/resources/expected/single/basic_api_with_mode.json b/integration/resources/expected/single/basic_api_with_mode.json index fda128f59..3f83d75c4 100644 --- a/integration/resources/expected/single/basic_api_with_mode.json +++ b/integration/resources/expected/single/basic_api_with_mode.json @@ -1,6 +1,6 @@ [ {"LogicalResourceId": "MyApi", "ResourceType": "AWS::ApiGateway::RestApi"}, - {"LogicalResourceId": "MyApiDeploymenta808f15210", "ResourceType": "AWS::ApiGateway::Deployment"}, + {"LogicalResourceId": "MyApiDeployment", "ResourceType": "AWS::ApiGateway::Deployment"}, {"LogicalResourceId": "MyApiMyNewStageNameStage", "ResourceType": "AWS::ApiGateway::Stage"}, {"LogicalResourceId": "TestFunction", "ResourceType": "AWS::Lambda::Function"}, {"LogicalResourceId": "TestFunctionAliaslive", "ResourceType": "AWS::Lambda::Alias"}, diff --git a/integration/resources/templates/combination/all_policy_templates.yaml b/integration/resources/templates/combination/all_policy_templates.yaml index 0b3fa6c55..a781aaa8e 100644 --- a/integration/resources/templates/combination/all_policy_templates.yaml +++ b/integration/resources/templates/combination/all_policy_templates.yaml @@ -8,7 +8,7 @@ Resources: Properties: CodeUri: ${codeuri} Handler: hello.handler - Runtime: python2.7 + Runtime: python3.8 Policies: - SQSPollerPolicy: @@ -123,7 +123,7 @@ Resources: Properties: CodeUri: ${codeuri} Handler: hello.handler - Runtime: python2.7 + Runtime: python3.8 Policies: - SESEmailTemplateCrudPolicy: {} @@ -187,7 +187,7 @@ Resources: Properties: CodeUri: ${codeuri} Handler: hello.handler - Runtime: python2.7 + Runtime: python3.8 Policies: - ElasticMapReduceModifyInstanceFleetPolicy: ClusterId: name diff --git a/integration/resources/templates/combination/depends_on.yaml b/integration/resources/templates/combination/depends_on.yaml index 2aeb01c18..a3dba05bf 100644 --- a/integration/resources/templates/combination/depends_on.yaml +++ b/integration/resources/templates/combination/depends_on.yaml @@ -12,7 +12,7 @@ Resources: Role: "Fn::GetAtt": LambdaRole.Arn Handler: lambda_function.lambda_handler - Runtime: python2.7 + Runtime: python3.8 Timeout: 15 CodeUri: ${codeuri} diff --git a/integration/resources/templates/combination/function_with_all_event_types.yaml b/integration/resources/templates/combination/function_with_all_event_types.yaml index 5f96c3fb5..7e5cc56bf 100644 --- a/integration/resources/templates/combination/function_with_all_event_types.yaml +++ b/integration/resources/templates/combination/function_with_all_event_types.yaml @@ -1,4 +1,9 @@ AWSTemplateFormatVersion: '2010-09-09' + +Parameters: + ScheduleName: + Type: String + Conditions: MyCondition: Fn::Equals: @@ -39,14 +44,7 @@ Resources: Properties: Schedule: 'rate(1 minute)' Name: - Fn::Sub: - - TestSchedule${__StackName__} - - __StackName__: - Fn::Select: - - 3 - - Fn::Split: - - "-" - - Ref: AWS::StackName + Ref: ScheduleName Description: test schedule Enabled: False @@ -142,16 +140,3 @@ Resources: WriteCapacityUnits: 5 StreamSpecification: StreamViewType: NEW_IMAGE - -Outputs: - ScheduleName: - Description: "Name of the cw schedule" - Value: - Fn::Sub: - - TestSchedule${__StackName__} - - __StackName__: - Fn::Select: - - 3 - - Fn::Split: - - "-" - - Ref: AWS::StackName \ No newline at end of file diff --git a/integration/resources/templates/combination/function_with_custom_code_deploy.yaml b/integration/resources/templates/combination/function_with_custom_code_deploy.yaml index 73b3dfcfc..0be5b828e 100644 --- a/integration/resources/templates/combination/function_with_custom_code_deploy.yaml +++ b/integration/resources/templates/combination/function_with_custom_code_deploy.yaml @@ -5,7 +5,7 @@ Resources: Properties: CodeUri: ${codeuri} Handler: index.handler - Runtime: python2.7 + Runtime: python3.8 AutoPublishAlias: Live diff --git a/integration/resources/templates/combination/function_with_deployment_basic.yaml b/integration/resources/templates/combination/function_with_deployment_basic.yaml index 1d40b0087..b89c09ff3 100644 --- a/integration/resources/templates/combination/function_with_deployment_basic.yaml +++ b/integration/resources/templates/combination/function_with_deployment_basic.yaml @@ -5,7 +5,7 @@ Resources: Properties: CodeUri: ${codeuri} Handler: index.handler - Runtime: python2.7 + Runtime: python3.8 AutoPublishAlias: Live diff --git a/integration/resources/templates/combination/function_with_deployment_default_role_managed_policy.yaml b/integration/resources/templates/combination/function_with_deployment_default_role_managed_policy.yaml index 1a627a6a3..9d8e637c9 100644 --- a/integration/resources/templates/combination/function_with_deployment_default_role_managed_policy.yaml +++ b/integration/resources/templates/combination/function_with_deployment_default_role_managed_policy.yaml @@ -4,7 +4,7 @@ Resources: Properties: CodeUri: ${codeuri} Handler: index.handler - Runtime: python2.7 + Runtime: python3.8 AutoPublishAlias: Live DeploymentPreference: Type: Canary10Percent5Minutes diff --git a/integration/resources/templates/combination/function_with_deployment_globals.yaml b/integration/resources/templates/combination/function_with_deployment_globals.yaml index 99b280a02..03adf56ca 100644 --- a/integration/resources/templates/combination/function_with_deployment_globals.yaml +++ b/integration/resources/templates/combination/function_with_deployment_globals.yaml @@ -16,7 +16,7 @@ Resources: Properties: CodeUri: ${codeuri} Handler: index.handler - Runtime: python2.7 + Runtime: python3.8 AutoPublishAlias: Live DeploymentRole: diff --git a/integration/resources/templates/combination/function_with_mq.yaml b/integration/resources/templates/combination/function_with_mq.yaml index 703b6c696..2c3483dc1 100644 --- a/integration/resources/templates/combination/function_with_mq.yaml +++ b/integration/resources/templates/combination/function_with_mq.yaml @@ -12,56 +12,40 @@ Parameters: MinLength: 12 ConstraintDescription: The Amazon MQ broker password is required ! NoEcho: true + PreCreatedVpc: + Type: String + PreCreatedSubnetOne: + Type: String + MQBrokerUserSecretName: + Type: String + PreCreatedInternetGateway: + Type: String + MQBrokerName: + Description: The name of MQ Broker + Type: String + Default: TestMQBroker Resources: - MyVpc: - Type: AWS::EC2::VPC - Properties: - CidrBlock: "10.42.0.0/16" - DependsOn: - - MyLambdaExecutionRole - - InternetGateway: - Type: AWS::EC2::InternetGateway - - AttachGateway: - Type: AWS::EC2::VPCGatewayAttachment - Properties: - VpcId: - Ref: MyVpc - InternetGatewayId: - Ref: InternetGateway RouteTable: Type: AWS::EC2::RouteTable Properties: VpcId: - Ref: MyVpc + Ref: PreCreatedVpc Route: Type: AWS::EC2::Route - DependsOn: AttachGateway Properties: RouteTableId: Ref: RouteTable DestinationCidrBlock: '0.0.0.0/0' GatewayId: - Ref: InternetGateway - PublicSubnet: - Type: AWS::EC2::Subnet - Properties: - VpcId: - Ref: MyVpc - CidrBlock: "10.42.0.0/24" - AvailabilityZone: - Fn::Select: - - 0 - - Fn::GetAZs: "" + Ref: PreCreatedInternetGateway PublicSubnetRouteTableAssociation: Type: AWS::EC2::SubnetRouteTableAssociation Properties: SubnetId: - Ref: PublicSubnet + Ref: PreCreatedSubnetOne RouteTableId: Ref: RouteTable @@ -71,7 +55,7 @@ Resources: GroupDescription: Limits security group ingress and egress traffic for the Amazon MQ instance VpcId: - Ref: MyVpc + Ref: PreCreatedVpc SecurityGroupIngress: - IpProtocol: tcp FromPort: 8162 @@ -134,7 +118,8 @@ Resources: MyMqBroker: Properties: - BrokerName: TestMQBroker + BrokerName: + Ref: MQBrokerName DeploymentMode: SINGLE_INSTANCE EngineType: ACTIVEMQ EngineVersion: 5.15.12 @@ -147,7 +132,7 @@ Resources: SecurityGroups: - Ref: MQSecurityGroup SubnetIds: - - Ref: PublicSubnet + - Ref: PreCreatedSubnetOne Users: - ConsoleAccess: true Groups: @@ -157,6 +142,7 @@ Resources: Password: Ref: MQBrokerPassword Type: AWS::AmazonMQ::Broker + DependsOn: MyLambdaExecutionRole MyLambdaFunction: Type: AWS::Serverless::Function @@ -182,7 +168,8 @@ Resources: MQBrokerUserSecret: Type: AWS::SecretsManager::Secret Properties: - Name: MQBrokerUserPassword + Name: + Ref: MQBrokerUserSecretName SecretString: Fn::Sub: '{"username":"${MQBrokerUser}","password":"${MQBrokerPassword}"}' Description: SecretsManager Secret for broker user and password \ No newline at end of file diff --git a/integration/resources/templates/combination/function_with_mq_using_autogen_role.yaml b/integration/resources/templates/combination/function_with_mq_using_autogen_role.yaml index 962bee9d9..e48757b15 100644 --- a/integration/resources/templates/combination/function_with_mq_using_autogen_role.yaml +++ b/integration/resources/templates/combination/function_with_mq_using_autogen_role.yaml @@ -12,54 +12,40 @@ Parameters: MinLength: 12 ConstraintDescription: The Amazon MQ broker password is required ! NoEcho: true + PreCreatedVpc: + Type: String + PreCreatedSubnetTwo: + Type: String + MQBrokerUserSecretName2: + Type: String + PreCreatedInternetGateway: + Type: String + MQBrokerName2: + Description: The name of MQ Broker + Type: String + Default: TestMQBroker2 Resources: - MyVpc: - Type: AWS::EC2::VPC - Properties: - CidrBlock: "10.42.0.0/16" - - InternetGateway: - Type: AWS::EC2::InternetGateway - - AttachGateway: - Type: AWS::EC2::VPCGatewayAttachment - Properties: - VpcId: - Ref: MyVpc - InternetGatewayId: - Ref: InternetGateway RouteTable: Type: AWS::EC2::RouteTable Properties: VpcId: - Ref: MyVpc + Ref: PreCreatedVpc Route: Type: AWS::EC2::Route - DependsOn: AttachGateway Properties: RouteTableId: Ref: RouteTable DestinationCidrBlock: '0.0.0.0/0' GatewayId: - Ref: InternetGateway - PublicSubnet: - Type: AWS::EC2::Subnet - Properties: - VpcId: - Ref: MyVpc - CidrBlock: "10.42.0.0/24" - AvailabilityZone: - Fn::Select: - - 0 - - Fn::GetAZs: "" + Ref: PreCreatedInternetGateway PublicSubnetRouteTableAssociation: Type: AWS::EC2::SubnetRouteTableAssociation Properties: SubnetId: - Ref: PublicSubnet + Ref: PreCreatedSubnetTwo RouteTableId: Ref: RouteTable @@ -69,7 +55,7 @@ Resources: GroupDescription: Limits security group ingress and egress traffic for the Amazon MQ instance VpcId: - Ref: MyVpc + Ref: PreCreatedVpc SecurityGroupIngress: - IpProtocol: tcp FromPort: 8162 @@ -94,7 +80,8 @@ Resources: MyMqBroker: Properties: - BrokerName: TestMQBroker2 + BrokerName: + Ref: MQBrokerName2 DeploymentMode: SINGLE_INSTANCE EngineType: ACTIVEMQ EngineVersion: 5.15.12 @@ -107,7 +94,7 @@ Resources: SecurityGroups: - Ref: MQSecurityGroup SubnetIds: - - Ref: PublicSubnet + - Ref: PreCreatedSubnetTwo Users: - ConsoleAccess: true Groups: @@ -140,7 +127,8 @@ Resources: MQBrokerUserSecret: Type: AWS::SecretsManager::Secret Properties: - Name: MQBrokerUserPassword2 + Name: + Ref: MQBrokerUserSecretName2 SecretString: Fn::Sub: '{"username":"${MQBrokerUser}","password":"${MQBrokerPassword}"}' Description: SecretsManager Secret for broker user and password diff --git a/integration/resources/templates/combination/function_with_msk.yaml b/integration/resources/templates/combination/function_with_msk.yaml index acaf3a383..9c4483e6c 100644 --- a/integration/resources/templates/combination/function_with_msk.yaml +++ b/integration/resources/templates/combination/function_with_msk.yaml @@ -1,37 +1,12 @@ -Resources: - MyVpc: - Type: "AWS::EC2::VPC" - Properties: - CidrBlock: "10.0.0.0/16" - DependsOn: - - MyLambdaExecutionRole - - MySubnetOne: - Type: "AWS::EC2::Subnet" - Properties: - VpcId: - Ref: MyVpc - CidrBlock: "10.0.0.0/24" - AvailabilityZone: - Fn::Select: - - 0 - - Fn::GetAZs: "" - DependsOn: - - MyVpc - - MySubnetTwo: - Type: "AWS::EC2::Subnet" - Properties: - VpcId: - Ref: MyVpc - CidrBlock: "10.0.1.0/24" - AvailabilityZone: - Fn::Select: - - 1 - - Fn::GetAZs: "" - DependsOn: - - MyVpc +Parameters: + PreCreatedSubnetOne: + Type: String + PreCreatedSubnetTwo: + Type: String + MskClusterName: + Type: String +Resources: MyLambdaExecutionRole: Type: AWS::IAM::Role Properties: @@ -68,20 +43,16 @@ Resources: Properties: BrokerNodeGroupInfo: ClientSubnets: - - Ref: MySubnetOne - - Ref: MySubnetTwo + - Ref: PreCreatedSubnetOne + - Ref: PreCreatedSubnetTwo InstanceType: kafka.t3.small StorageInfo: EBSStorageInfo: VolumeSize: 1 - ClusterName: MyMskClusterTestName + ClusterName: + Ref: MskClusterName KafkaVersion: 2.4.1.1 NumberOfBrokerNodes: 2 - DependsOn: - - MyVpc - - MySubnetOne - - MySubnetTwo - - MyLambdaExecutionRole MyMskStreamProcessor: Type: AWS::Serverless::Function @@ -96,14 +67,8 @@ Resources: Type: MSK Properties: StartingPosition: LATEST - Stream: + Stream: Ref: MyMskCluster Topics: - "MyDummyTestTopic" - DependsOn: - - MyVpc - - MySubnetOne - - MySubnetTwo - - MyLambdaExecutionRole - - MyMskCluster diff --git a/integration/resources/templates/combination/function_with_msk_using_managed_policy.yaml b/integration/resources/templates/combination/function_with_msk_using_managed_policy.yaml index 0fa07ba4a..8ee3b645d 100644 --- a/integration/resources/templates/combination/function_with_msk_using_managed_policy.yaml +++ b/integration/resources/templates/combination/function_with_msk_using_managed_policy.yaml @@ -1,53 +1,27 @@ -Resources: - MyVpc: - Type: "AWS::EC2::VPC" - Properties: - CidrBlock: "10.0.0.0/16" - - MySubnetOne: - Type: "AWS::EC2::Subnet" - Properties: - VpcId: - Ref: MyVpc - CidrBlock: "10.0.0.0/24" - AvailabilityZone: - Fn::Select: - - 0 - - Fn::GetAZs: "" - DependsOn: - - MyVpc - - MySubnetTwo: - Type: "AWS::EC2::Subnet" - Properties: - VpcId: - Ref: MyVpc - CidrBlock: "10.0.1.0/24" - AvailabilityZone: - Fn::Select: - - 1 - - Fn::GetAZs: "" - DependsOn: - - MyVpc +Parameters: + PreCreatedSubnetOne: + Type: String + PreCreatedSubnetTwo: + Type: String + MskClusterName2: + Type: String +Resources: MyMskCluster: Type: 'AWS::MSK::Cluster' Properties: BrokerNodeGroupInfo: ClientSubnets: - - Ref: MySubnetOne - - Ref: MySubnetTwo + - Ref: PreCreatedSubnetOne + - Ref: PreCreatedSubnetTwo InstanceType: kafka.t3.small StorageInfo: EBSStorageInfo: VolumeSize: 1 - ClusterName: MyMskClusterTestName2 + ClusterName: + Ref: MskClusterName2 KafkaVersion: 2.4.1.1 NumberOfBrokerNodes: 2 - DependsOn: - - MyVpc - - MySubnetOne - - MySubnetTwo MyMskStreamProcessor: Type: AWS::Serverless::Function @@ -60,13 +34,8 @@ Resources: Type: MSK Properties: StartingPosition: LATEST - Stream: + Stream: Ref: MyMskCluster Topics: - "MyDummyTestTopic" - DependsOn: - - MyVpc - - MySubnetOne - - MySubnetTwo - - MyMskCluster diff --git a/integration/resources/templates/combination/function_with_policy_templates.yaml b/integration/resources/templates/combination/function_with_policy_templates.yaml index bd0cec1c0..31bcc1a30 100644 --- a/integration/resources/templates/combination/function_with_policy_templates.yaml +++ b/integration/resources/templates/combination/function_with_policy_templates.yaml @@ -14,7 +14,7 @@ Resources: Properties: CodeUri: ${codeuri} Handler: hello.handler - Runtime: python2.7 + Runtime: python3.8 Policies: - SQSPollerPolicy: QueueName: diff --git a/integration/resources/templates/combination/function_with_resource_refs.yaml b/integration/resources/templates/combination/function_with_resource_refs.yaml index 1d091fe91..d8b92b71b 100644 --- a/integration/resources/templates/combination/function_with_resource_refs.yaml +++ b/integration/resources/templates/combination/function_with_resource_refs.yaml @@ -10,14 +10,14 @@ Resources: Properties: CodeUri: ${codeuri} Handler: hello.handler - Runtime: python2.7 + Runtime: python3.8 AutoPublishAlias: Live MyOtherFunction: Type: 'AWS::Serverless::Function' Properties: CodeUri: ${codeuri} - Runtime: python2.7 + Runtime: python3.8 Handler: hello.handler Environment: Variables: diff --git a/integration/resources/templates/combination/function_with_schedule.yaml b/integration/resources/templates/combination/function_with_schedule.yaml index 97cda940b..42e352db8 100644 --- a/integration/resources/templates/combination/function_with_schedule.yaml +++ b/integration/resources/templates/combination/function_with_schedule.yaml @@ -1,3 +1,7 @@ +Parameters: + ScheduleName: + Type: String + Resources: MyLambdaFunction: Type: AWS::Serverless::Function @@ -13,25 +17,6 @@ Resources: Schedule: 'rate(5 minutes)' Input: '{"Hello": "world!"}' Name: - Fn::Sub: - - TestSchedule${__StackName__} - - __StackName__: - Fn::Select: - - 3 - - Fn::Split: - - "-" - - Ref: AWS::StackName + Ref: ScheduleName Description: test schedule Enabled: True -Outputs: - ScheduleName: - Description: "Name of the cw schedule" - Value: - Fn::Sub: - - TestSchedule${__StackName__} - - __StackName__: - Fn::Select: - - 3 - - Fn::Split: - - "-" - - Ref: AWS::StackName \ No newline at end of file diff --git a/integration/resources/templates/combination/state_machine_with_policy_templates.yaml b/integration/resources/templates/combination/state_machine_with_policy_templates.yaml index 5d1ed150d..cafa9f007 100644 --- a/integration/resources/templates/combination/state_machine_with_policy_templates.yaml +++ b/integration/resources/templates/combination/state_machine_with_policy_templates.yaml @@ -24,7 +24,7 @@ Resources: Properties: CodeUri: ${codeuri} Handler: hello.handler - Runtime: python2.7 + Runtime: python3.8 MyQueue: Type: AWS::SQS::Queue diff --git a/integration/resources/templates/single/basic_application_sar_location.yaml b/integration/resources/templates/single/basic_application_sar_location.yaml index 5bd59c2e4..254eca341 100644 --- a/integration/resources/templates/single/basic_application_sar_location.yaml +++ b/integration/resources/templates/single/basic_application_sar_location.yaml @@ -3,7 +3,7 @@ Resources: Type: AWS::Serverless::Application Properties: Location: - ApplicationId: arn:aws:serverlessrepo:us-east-1:077246666028:applications/hello-world-python + ApplicationId: arn:aws:serverlessrepo:us-east-1:077246666028:applications/hello-world-python3 SemanticVersion: 1.0.2 Parameters: IdentityNameParameter: test diff --git a/integration/resources/templates/single/basic_application_sar_location_with_intrinsics.yaml b/integration/resources/templates/single/basic_application_sar_location_with_intrinsics.yaml index 471af7ae0..3de30267d 100644 --- a/integration/resources/templates/single/basic_application_sar_location_with_intrinsics.yaml +++ b/integration/resources/templates/single/basic_application_sar_location_with_intrinsics.yaml @@ -6,7 +6,7 @@ Parameters: Mappings: SARApplication: us-east-1: - ApplicationId: arn:aws:serverlessrepo:us-east-1:077246666028:applications/hello-world-python + ApplicationId: arn:aws:serverlessrepo:us-east-1:077246666028:applications/hello-world-python3 us-east-2: ApplicationId: arn:aws:serverlessrepo:us-east-1:077246666028:applications/hello-world-python3 us-west-1: diff --git a/integration/resources/templates/single/basic_function_event_destinations.yaml b/integration/resources/templates/single/basic_function_event_destinations.yaml index 2129313b1..d3f8d9934 100644 --- a/integration/resources/templates/single/basic_function_event_destinations.yaml +++ b/integration/resources/templates/single/basic_function_event_destinations.yaml @@ -42,7 +42,7 @@ Resources: } }; Handler: index.handler - Runtime: nodejs10.x + Runtime: nodejs14.x MemorySize: 1024 MyTestFunction2: Type: AWS::Serverless::Function @@ -74,7 +74,7 @@ Resources: } }; Handler: index.handler - Runtime: nodejs10.x + Runtime: nodejs14.x MemorySize: 1024 DestinationLambda: Type: AWS::Serverless::Function @@ -88,7 +88,7 @@ Resources: return response; }; Handler: index.handler - Runtime: nodejs10.x + Runtime: nodejs14.x MemorySize: 1024 DestinationSQS: Condition: QueueCreationDisabled diff --git a/integration/resources/templates/single/basic_layer_with_parameters.yaml b/integration/resources/templates/single/basic_layer_with_parameters.yaml index f5eb8fdb8..e347b96e1 100644 --- a/integration/resources/templates/single/basic_layer_with_parameters.yaml +++ b/integration/resources/templates/single/basic_layer_with_parameters.yaml @@ -7,7 +7,7 @@ Parameters: Default: MIT-0 Runtimes: Type: CommaDelimitedList - Default: nodejs12.x,nodejs10.x + Default: nodejs12.x,nodejs14.x LayerName: Type: String Default: MyNamedLayerVersion diff --git a/integration/resources/templates/single/function_with_deployment_preference_alarms_intrinsic_if.yaml b/integration/resources/templates/single/function_with_deployment_preference_alarms_intrinsic_if.yaml index 261571ff5..ddc1a13bd 100644 --- a/integration/resources/templates/single/function_with_deployment_preference_alarms_intrinsic_if.yaml +++ b/integration/resources/templates/single/function_with_deployment_preference_alarms_intrinsic_if.yaml @@ -9,7 +9,7 @@ Resources: Properties: CodeUri: ${codeuri} Handler: hello.handler - Runtime: python2.7 + Runtime: python3.8 AutoPublishAlias: live DeploymentPreference: Type: Linear10PercentEvery3Minutes diff --git a/integration/setup/__init__.py b/integration/setup/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/integration/setup/companion-stack.yaml b/integration/setup/companion-stack.yaml new file mode 100644 index 000000000..35521d7f2 --- /dev/null +++ b/integration/setup/companion-stack.yaml @@ -0,0 +1,60 @@ +Resources: + PreCreatedVpc: + Type: "AWS::EC2::VPC" + Properties: + CidrBlock: "10.0.0.0/16" + + PreCreatedSubnetOne: + Type: "AWS::EC2::Subnet" + Properties: + VpcId: + Ref: PreCreatedVpc + CidrBlock: "10.0.0.0/24" + AvailabilityZone: + Fn::Select: + - 0 + - Fn::GetAZs: "" + + PreCreatedSubnetTwo: + Type: "AWS::EC2::Subnet" + Properties: + VpcId: + Ref: PreCreatedVpc + CidrBlock: "10.0.1.0/24" + AvailabilityZone: + Fn::Select: + - 1 + - Fn::GetAZs: "" + + PreCreatedInternetGateway: + Type: AWS::EC2::InternetGateway + + PreCreatedAttachGateway: + Type: AWS::EC2::VPCGatewayAttachment + Properties: + VpcId: + Ref: PreCreatedVpc + InternetGatewayId: + Ref: PreCreatedInternetGateway + +Outputs: + PreCreatedVpc: + Description: "Pre-created VPC that can be used inside other tests" + Value: + Ref: PreCreatedVpc + PreCreatedSubnetTwo: + Description: "Pre-created #2 subnet that can be used inside other tests" + Value: + Ref: PreCreatedSubnetTwo + PreCreatedSubnetOne: + Description: "Pre-created #1 subnet that can be used inside other tests" + Value: + Ref: PreCreatedSubnetOne + PreCreatedInternetGateway: + Description: "Pre-created Internet Gateway that can be used inside other tests" + Value: + Ref: PreCreatedInternetGateway + PreCreatedAttachGateway: + Description: "Pre-created Attach Gateway that can be used inside other tests" + Value: + Ref: PreCreatedAttachGateway \ No newline at end of file diff --git a/integration/setup/test_setup_teardown.py b/integration/setup/test_setup_teardown.py new file mode 100644 index 000000000..49ba6aa96 --- /dev/null +++ b/integration/setup/test_setup_teardown.py @@ -0,0 +1,11 @@ +import pytest + + +@pytest.mark.setup +def test_setup(setup_companion_stack_once): + assert True + + +@pytest.mark.teardown +def test_teardown(delete_companion_stack_once): + assert True diff --git a/integration/single/test_basic_api.py b/integration/single/test_basic_api.py index 846d96718..e8de40556 100644 --- a/integration/single/test_basic_api.py +++ b/integration/single/test_basic_api.py @@ -1,6 +1,12 @@ +import time +from unittest.case import skipIf + from integration.helpers.base_test import BaseTest import requests +from integration.helpers.resource import current_region_does_not_support +from integration.config.service_names import MODE + class TestBasicApi(BaseTest): """ @@ -25,6 +31,7 @@ def test_basic_api(self): self.assertEqual(len(set(first_dep_ids).intersection(second_dep_ids)), 0) + @skipIf(current_region_does_not_support([MODE]), "Mode is not supported in this testing region") def test_basic_api_with_mode(self): """ Creates an API and updates its DefinitionUri @@ -39,8 +46,16 @@ def test_basic_api_with_mode(self): # Removes get from the API self.update_and_verify_stack("single/basic_api_with_mode_update") - response = requests.get(f"{api_endpoint}/get") + # API Gateway by default returns 403 if a path do not exist + retries = 20 + while retries > 0: + retries -= 1 + response = requests.get(f"{api_endpoint}/get") + if response.status_code != 500: + break + time.sleep(5) + self.assertEqual(response.status_code, 403) def test_basic_api_inline_openapi(self): diff --git a/integration/single/test_basic_application.py b/integration/single/test_basic_application.py index 15ae0861f..afc7a97c5 100644 --- a/integration/single/test_basic_application.py +++ b/integration/single/test_basic_application.py @@ -38,7 +38,7 @@ def test_basic_application_sar_location(self): functions = self.get_stack_resources("AWS::Lambda::Function", nested_stack_resource) self.assertEqual(len(functions), 1) - self.assertEqual(functions[0]["LogicalResourceId"], "helloworldpython") + self.assertEqual(functions[0]["LogicalResourceId"], "helloworldpython3") @skipIf( current_region_does_not_support(["ServerlessRepo"]), "ServerlessRepo is not supported in this testing region" @@ -47,7 +47,7 @@ def test_basic_application_sar_location_with_intrinsics(self): """ Creates an application with a lambda function with intrinsics """ - expected_function_name = "helloworldpython" if self.get_region() == "us-east-1" else "helloworldpython3" + expected_function_name = "helloworldpython3" self.create_and_verify_stack("single/basic_application_sar_location_with_intrinsics") nested_stack_resource = self.get_stack_nested_stack_resources()