Release/v1.36.0 (#2042) (#2050)

* refactor: Optimize shared API usage plan handling (#1973)

* fix: use instance variables for generating shared api usage plan

* add extra log statements

* fix: Added SAR Support Check (#1972)

* Added SAR Support Check

* Added docstring and Removed Instance Initialization for Class Method

* set log level explicitly

* update pyyaml version to get the security update (#1974)

* fix: use instance variables for generating shared api usage plan

* add extra log statements

* set log level explicitly

* black formatting

* black formatting

Co-authored-by: Cosh_ <[email protected]>
Co-authored-by: Mohamed Elasmar <[email protected]>

* feat: Resource level attributes support (#2008)

* Fix for invalid MQ event source managed policy

* Fix for invalid managed policy for MQ, included support for new MQ event source property, updated test cases

* Black reformatting

* Test case changes

* Changed policy name

* Modified test cases with new policy name

* Added resource attributes and unit tests

* Resource attributes initial work

* Passthrough attributes for some resources, updated some tests

* Resolve merge conflicts

* Fixed a typo

* Modified implicit api plugin for resource attributes support

* Partial update of the tests

* Partially updated test cases, black reformatted

* Partially updated test templates

* Partially updated test templates

* Partially updated test templates

* Added event bridge support for passthrough resource attributes

* Partially updated test templates (up to function with amq kms)

* Partially updated test templates (up to sns)

* Partially updated test templates (all the ones left)

* Prevented passthrough resource attributes from changing layer version hashes

* Added test to verify resource passthrough precedence for implicit api

* Modified tests related to lambda layer to revert the hash changes, keeping the hash the same with resource attributes added

* fix: mutable default values in method definitions (#1997)

* fix: remove explicit logging level set in single module (#1998)

* run automated tests for resource level attribute support

* Skipping metadata in layer hashing

* Refactored the classes for TestTranslatorEndToEnd and TestResourceLevelAttributes to share the same parent class

* Added new translator tests for version and layer resources

* Added new unit tests

* Removed after transform resource plugin

* Black reformatting

* Refactoring implicit api plugin support for DeletionPolicy and UpdateReplacePolicy

* Refactoring to improve code quality

* Added simple documentation

* Black reformatting

* Added input template that was missing

* Refactoring: use sets instead of lists for implicit api plugin

* Changing import to be compatible with py2.7

* Changing test deployment hashes to their actual values

Co-authored-by: Mehmet Nuri Deveci <[email protected]>

* chore: bump version to 1.36.0 (#2014)

* fix: Shared Usage Plan scenarios for Resource Level Attribute Support (#2040)

* do not propagate attributes to CognitoUserPool

* shared usage plan with propagated resource level attributes

* add unit tests

* Added test templates for shared usage plan with resource attributes

* Black reformatting

* Removing unused import

* fix python2 hashes

Co-authored-by: Qingchuan Ma <[email protected]>

Co-authored-by: Mehmet Nuri Deveci <[email protected]>
Co-authored-by: Cosh_ <[email protected]>
Co-authored-by: Mohamed Elasmar <[email protected]>

Co-authored-by: Mehmet Nuri Deveci <[email protected]>
Co-authored-by: Cosh_ <[email protected]>
Co-authored-by: Mohamed Elasmar <[email protected]>

Author: 4 people

samtranslator/__init__.py

@@ -1 +1 @@
-__version__ = "1.35.0"
+__version__ = "1.36.0"

samtranslator/model/__init__.py

@@ -43,7 +43,11 @@ class Resource(object):
     property_types = None
     _keywords = ["logical_id", "relative_id", "depends_on", "resource_attributes"]
 
-    _supported_resource_attributes = ["DeletionPolicy", "UpdatePolicy", "Condition"]
+    # For attributes in this list, they will be passed into the translated template for the same resource itself.
+    _supported_resource_attributes = ["DeletionPolicy", "UpdatePolicy", "Condition", "UpdateReplacePolicy", "Metadata"]
+    # For attributes in this list, they will be passed into the translated template for the same resource,
+    # as well as all the auto-generated resources that are created from this resource.
+    _pass_through_attributes = ["Condition", "DeletionPolicy", "UpdateReplacePolicy"]
 
     # Runtime attributes that can be qureied resource. They are CloudFormation attributes like ARN, Name etc that
     # will be resolvable at runtime. This map will be implemented by sub-classes to express list of attributes they
@@ -76,6 +80,22 @@ def __init__(self, logical_id, relative_id=None, depends_on=None, attributes=Non
             for attr, value in attributes.items():
                 self.set_resource_attribute(attr, value)
 
+    @classmethod
+    def get_supported_resource_attributes(cls):
+        """
+        A getter method for the supported resource attributes
+        returns: a tuple that contains the name of all supported resource attributes
+        """
+        return tuple(cls._supported_resource_attributes)
+
+    @classmethod
+    def get_pass_through_attributes(cls):
+        """
+        A getter method for the resource attributes to be passed to auto-generated resources
+        returns: a tuple that contains the name of all pass through attributes
+        """
+        return tuple(cls._pass_through_attributes)
+
     @classmethod
     def from_dict(cls, logical_id, resource_dict, relative_id=None, sam_plugins=None):
         """Constructs a Resource object with the given logical id, based on the given resource dict. The resource dict
@@ -318,9 +338,10 @@ def get_passthrough_resource_attributes(self):
 
         :return: Dictionary of resource attributes.
         """
-        attributes = None
-        if "Condition" in self.resource_attributes:
-            attributes = {"Condition": self.resource_attributes["Condition"]}
+        attributes = {}
+        for resource_attribute in self.get_pass_through_attributes():
+            if resource_attribute in self.resource_attributes:
+                attributes[resource_attribute] = self.resource_attributes.get(resource_attribute)
         return attributes
 
 

samtranslator/model/api/api_generator.py

@@ -1,6 +1,8 @@
+import logging
 from collections import namedtuple
+
 from six import string_types
-from samtranslator.model.intrinsics import ref, fnGetAtt
+from samtranslator.model.intrinsics import ref, fnGetAtt, make_or_condition
 from samtranslator.model.apigateway import (
     ApiGatewayDeployment,
     ApiGatewayRestApi,
@@ -14,7 +16,7 @@
     ApiGatewayApiKey,
 )
 from samtranslator.model.route53 import Route53RecordSetGroup
-from samtranslator.model.exceptions import InvalidResourceException
+from samtranslator.model.exceptions import InvalidResourceException, InvalidTemplateException
 from samtranslator.model.s3_utils.uri_parser import parse_s3_uri
 from samtranslator.region_configuration import RegionConfiguration
 from samtranslator.swagger.swagger import SwaggerEditor
@@ -24,6 +26,9 @@
 from samtranslator.translator.arn_generator import ArnGenerator
 from samtranslator.model.tags.resource_tagging import get_tag_list
 
+LOG = logging.getLogger(__name__)
+LOG.setLevel(logging.INFO)
+
 _CORS_WILDCARD = "'*'"
 CorsProperties = namedtuple(
     "_CorsProperties", ["AllowMethods", "AllowHeaders", "AllowOrigin", "MaxAge", "AllowCredentials"]
@@ -52,12 +57,100 @@
 GatewayResponseProperties = ["ResponseParameters", "ResponseTemplates", "StatusCode"]
 
 
-class ApiGenerator(object):
-    usage_plan_shared = False
-    stage_keys_shared = list()
-    api_stages_shared = list()
-    depends_on_shared = list()
+class SharedApiUsagePlan(object):
+    """
+    Collects API information from different API resources in the same template,
+    so that these information can be used in the shared usage plan
+    """
+
+    SHARED_USAGE_PLAN_CONDITION_NAME = "SharedUsagePlanCondition"
+
+    def __init__(self):
+        self.usage_plan_shared = False
+        self.stage_keys_shared = list()
+        self.api_stages_shared = list()
+        self.depends_on_shared = list()
+
+        # shared resource level attributes
+        self.conditions = set()
+        self.any_api_without_condition = False
+        self.deletion_policy = None
+        self.update_replace_policy = None
+
+    def get_combined_resource_attributes(self, resource_attributes, conditions):
+        """
+        This method returns a dictionary which combines 'DeletionPolicy', 'UpdateReplacePolicy' and 'Condition'
+        values of API definitions that could be used in Shared Usage Plan resources.
+
+        Parameters
+        ----------
+        resource_attributes: Dict[str]
+            A dictionary of resource level attributes of the API resource
+        conditions: Dict[str]
+            Conditions section of the template
+        """
+        self._set_deletion_policy(resource_attributes.get("DeletionPolicy"))
+        self._set_update_replace_policy(resource_attributes.get("UpdateReplacePolicy"))
+        self._set_condition(resource_attributes.get("Condition"), conditions)
+
+        combined_resource_attributes = dict()
+        if self.deletion_policy:
+            combined_resource_attributes["DeletionPolicy"] = self.deletion_policy
+        if self.update_replace_policy:
+            combined_resource_attributes["UpdateReplacePolicy"] = self.update_replace_policy
+        # do not set Condition if any of the API resource does not have Condition in it
+        if self.conditions and not self.any_api_without_condition:
+            combined_resource_attributes["Condition"] = SharedApiUsagePlan.SHARED_USAGE_PLAN_CONDITION_NAME
+
+        return combined_resource_attributes
+
+    def _set_deletion_policy(self, deletion_policy):
+        if deletion_policy:
+            if self.deletion_policy:
+                # update only if new deletion policy is Retain
+                if deletion_policy == "Retain":
+                    self.deletion_policy = deletion_policy
+            else:
+                self.deletion_policy = deletion_policy
+
+    def _set_update_replace_policy(self, update_replace_policy):
+        if update_replace_policy:
+            if self.update_replace_policy:
+                # if new value is Retain or
+                # new value is retain and current value is Delete then update its value
+                if (update_replace_policy == "Retain") or (
+                    update_replace_policy == "Snapshot" and self.update_replace_policy == "Delete"
+                ):
+                    self.update_replace_policy = update_replace_policy
+            else:
+                self.update_replace_policy = update_replace_policy
+
+    def _set_condition(self, condition, template_conditions):
+        # if there are any API without condition, then skip
+        if self.any_api_without_condition:
+            return
 
+        if condition and condition not in self.conditions:
+
+            if template_conditions is None:
+                raise InvalidTemplateException(
+                    "Can't have condition without having 'Conditions' section in the template"
+                )
+
+            if self.conditions:
+                self.conditions.add(condition)
+                or_condition = make_or_condition(self.conditions)
+                template_conditions[SharedApiUsagePlan.SHARED_USAGE_PLAN_CONDITION_NAME] = or_condition
+            else:
+                self.conditions.add(condition)
+                template_conditions[SharedApiUsagePlan.SHARED_USAGE_PLAN_CONDITION_NAME] = condition
+        elif condition is None:
+            self.any_api_without_condition = True
+            if template_conditions and SharedApiUsagePlan.SHARED_USAGE_PLAN_CONDITION_NAME in template_conditions:
+                del template_conditions[SharedApiUsagePlan.SHARED_USAGE_PLAN_CONDITION_NAME]
+
+
+class ApiGenerator(object):
     def __init__(
         self,
         logical_id,
@@ -69,6 +162,8 @@ def __init__(
         definition_uri,
         name,
         stage_name,
+        shared_api_usage_plan,
+        template_conditions,
         tags=None,
         endpoint_configuration=None,
         method_settings=None,
@@ -134,6 +229,8 @@ def __init__(
         self.models = models
         self.domain = domain
         self.description = description
+        self.shared_api_usage_plan = shared_api_usage_plan
+        self.template_conditions = template_conditions
 
     def _construct_rest_api(self):
         """Constructs and returns the ApiGateway RestApi.
@@ -617,7 +714,11 @@ def _construct_usage_plan(self, rest_api_stage=None):
         # create usage plan for this api only
         elif usage_plan_properties.get("CreateUsagePlan") == "PER_API":
             usage_plan_logical_id = self.logical_id + "UsagePlan"
-            usage_plan = ApiGatewayUsagePlan(logical_id=usage_plan_logical_id, depends_on=[self.logical_id])
+            usage_plan = ApiGatewayUsagePlan(
+                logical_id=usage_plan_logical_id,
+                depends_on=[self.logical_id],
+                attributes=self.passthrough_resource_attributes,
+            )
             api_stages = list()
             api_stage = dict()
             api_stage["ApiId"] = ref(self.logical_id)
@@ -630,18 +731,23 @@ def _construct_usage_plan(self, rest_api_stage=None):
 
         # create a usage plan for all the Apis
         elif create_usage_plan == "SHARED":
+            LOG.info("Creating SHARED usage plan for all the Apis")
             usage_plan_logical_id = "ServerlessUsagePlan"
-            if self.logical_id not in ApiGenerator.depends_on_shared:
-                ApiGenerator.depends_on_shared.append(self.logical_id)
+            if self.logical_id not in self.shared_api_usage_plan.depends_on_shared:
+                self.shared_api_usage_plan.depends_on_shared.append(self.logical_id)
             usage_plan = ApiGatewayUsagePlan(
-                logical_id=usage_plan_logical_id, depends_on=ApiGenerator.depends_on_shared
+                logical_id=usage_plan_logical_id,
+                depends_on=self.shared_api_usage_plan.depends_on_shared,
+                attributes=self.shared_api_usage_plan.get_combined_resource_attributes(
+                    self.passthrough_resource_attributes, self.template_conditions
+                ),
             )
             api_stage = dict()
             api_stage["ApiId"] = ref(self.logical_id)
             api_stage["Stage"] = ref(rest_api_stage.logical_id)
-            if api_stage not in ApiGenerator.api_stages_shared:
-                ApiGenerator.api_stages_shared.append(api_stage)
-            usage_plan.ApiStages = ApiGenerator.api_stages_shared
+            if api_stage not in self.shared_api_usage_plan.api_stages_shared:
+                self.shared_api_usage_plan.api_stages_shared.append(api_stage)
+            usage_plan.ApiStages = self.shared_api_usage_plan.api_stages_shared
 
             api_key = self._construct_api_key(usage_plan_logical_id, create_usage_plan, rest_api_stage)
             usage_plan_key = self._construct_usage_plan_key(usage_plan_logical_id, create_usage_plan, api_key)
@@ -667,20 +773,31 @@ def _construct_api_key(self, usage_plan_logical_id, create_usage_plan, rest_api_
         """
         if create_usage_plan == "SHARED":
             # create an api key resource for all the apis
+            LOG.info("Creating api key resource for all the Apis from SHARED usage plan")
             api_key_logical_id = "ServerlessApiKey"
-            api_key = ApiGatewayApiKey(logical_id=api_key_logical_id, depends_on=[usage_plan_logical_id])
+            api_key = ApiGatewayApiKey(
+                logical_id=api_key_logical_id,
+                depends_on=[usage_plan_logical_id],
+                attributes=self.shared_api_usage_plan.get_combined_resource_attributes(
+                    self.passthrough_resource_attributes, self.template_conditions
+                ),
+            )
             api_key.Enabled = True
             stage_key = dict()
             stage_key["RestApiId"] = ref(self.logical_id)
             stage_key["StageName"] = ref(rest_api_stage.logical_id)
-            if stage_key not in ApiGenerator.stage_keys_shared:
-                ApiGenerator.stage_keys_shared.append(stage_key)
-            api_key.StageKeys = ApiGenerator.stage_keys_shared
+            if stage_key not in self.shared_api_usage_plan.stage_keys_shared:
+                self.shared_api_usage_plan.stage_keys_shared.append(stage_key)
+            api_key.StageKeys = self.shared_api_usage_plan.stage_keys_shared
         # for create_usage_plan = "PER_API"
         else:
             # create an api key resource for this api
             api_key_logical_id = self.logical_id + "ApiKey"
-            api_key = ApiGatewayApiKey(logical_id=api_key_logical_id, depends_on=[usage_plan_logical_id])
+            api_key = ApiGatewayApiKey(
+                logical_id=api_key_logical_id,
+                depends_on=[usage_plan_logical_id],
+                attributes=self.passthrough_resource_attributes,
+            )
             api_key.Enabled = True
             stage_keys = list()
             stage_key = dict()
@@ -700,12 +817,20 @@ def _construct_usage_plan_key(self, usage_plan_logical_id, create_usage_plan, ap
         if create_usage_plan == "SHARED":
             # create a mapping between api key and the usage plan
             usage_plan_key_logical_id = "ServerlessUsagePlanKey"
+            resource_attributes = self.shared_api_usage_plan.get_combined_resource_attributes(
+                self.passthrough_resource_attributes, self.template_conditions
+            )
         # for create_usage_plan = "PER_API"
         else:
             # create a mapping between api key and the usage plan
             usage_plan_key_logical_id = self.logical_id + "UsagePlanKey"
+            resource_attributes = self.passthrough_resource_attributes
 
-        usage_plan_key = ApiGatewayUsagePlanKey(logical_id=usage_plan_key_logical_id, depends_on=[api_key.logical_id])
+        usage_plan_key = ApiGatewayUsagePlanKey(
+            logical_id=usage_plan_key_logical_id,
+            depends_on=[api_key.logical_id],
+            attributes=resource_attributes,
+        )
         usage_plan_key.KeyId = ref(api_key.logical_id)
         usage_plan_key.KeyType = "API_KEY"
         usage_plan_key.UsagePlanId = ref(usage_plan_logical_id)

samtranslator/model/eventbridge_utils.py

@@ -4,15 +4,15 @@
 
 class EventBridgeRuleUtils:
     @staticmethod
-    def create_dead_letter_queue_with_policy(rule_logical_id, rule_arn, queue_logical_id=None):
+    def create_dead_letter_queue_with_policy(rule_logical_id, rule_arn, queue_logical_id=None, attributes=None):
         resources = []
 
-        queue = SQSQueue(queue_logical_id or rule_logical_id + "Queue")
+        queue = SQSQueue(queue_logical_id or rule_logical_id + "Queue", attributes=attributes)
         dlq_queue_arn = queue.get_runtime_attr("arn")
         dlq_queue_url = queue.get_runtime_attr("queue_url")
 
         # grant necessary permission to Eventbridge Rule resource for sending messages to dead-letter queue
-        policy = SQSQueuePolicy(rule_logical_id + "QueuePolicy")
+        policy = SQSQueuePolicy(rule_logical_id + "QueuePolicy", attributes=attributes)
         policy.PolicyDocument = SQSQueuePolicies.eventbridge_dlq_send_message_resource_based_policy(
             rule_arn, dlq_queue_arn
         )
@@ -41,14 +41,14 @@ def validate_dlq_config(source_logical_id, dead_letter_config):
             raise InvalidEventException(source_logical_id, "No 'Arn' or 'Type' property provided for DeadLetterConfig")
 
     @staticmethod
-    def get_dlq_queue_arn_and_resources(cw_event_source, source_arn):
+    def get_dlq_queue_arn_and_resources(cw_event_source, source_arn, attributes):
         """returns dlq queue arn and dlq_resources, assuming cw_event_source.DeadLetterConfig has been validated"""
         dlq_queue_arn = cw_event_source.DeadLetterConfig.get("Arn")
         if dlq_queue_arn is not None:
             return dlq_queue_arn, []
         queue_logical_id = cw_event_source.DeadLetterConfig.get("QueueLogicalId")
         dlq_resources = EventBridgeRuleUtils.create_dead_letter_queue_with_policy(
-            cw_event_source.logical_id, source_arn, queue_logical_id
+            cw_event_source.logical_id, source_arn, queue_logical_id, attributes
         )
         dlq_queue_arn = dlq_resources[0].get_runtime_attr("arn")
         return dlq_queue_arn, dlq_resources

samtranslator/model/eventsources/cloudwatchlogs.py

@@ -43,11 +43,13 @@ def get_source_arn(self):
         )
 
     def get_subscription_filter(self, function, permission):
-        subscription_filter = SubscriptionFilter(self.logical_id, depends_on=[permission.logical_id])
+        subscription_filter = SubscriptionFilter(
+            self.logical_id,
+            depends_on=[permission.logical_id],
+            attributes=function.get_passthrough_resource_attributes(),
+        )
         subscription_filter.LogGroupName = self.LogGroupName
         subscription_filter.FilterPattern = self.FilterPattern
         subscription_filter.DestinationArn = function.get_runtime_attr("arn")
-        if "Condition" in function.resource_attributes:
-            subscription_filter.set_resource_attribute("Condition", function.resource_attributes["Condition"])
 
         return subscription_filter

samtranslator/model/eventsources/pull.py

@@ -60,7 +60,9 @@ def to_cloudformation(self, **kwargs):
 
         resources = []
 
-        lambda_eventsourcemapping = LambdaEventSourceMapping(self.logical_id)
+        lambda_eventsourcemapping = LambdaEventSourceMapping(
+            self.logical_id, attributes=function.get_passthrough_resource_attributes()
+        )
         resources.append(lambda_eventsourcemapping)
 
         try:
@@ -122,9 +124,6 @@ def to_cloudformation(self, **kwargs):
                     )
             lambda_eventsourcemapping.DestinationConfig = self.DestinationConfig
 
-        if "Condition" in function.resource_attributes:
-            lambda_eventsourcemapping.set_resource_attribute("Condition", function.resource_attributes["Condition"])
-
         if "role" in kwargs:
             self._link_policy(kwargs["role"], destination_config_policy)