diff --git a/samtranslator/plugins/__init__.py b/samtranslator/plugins/__init__.py index 320910028..2a565993f 100644 --- a/samtranslator/plugins/__init__.py +++ b/samtranslator/plugins/__init__.py @@ -1,152 +1,10 @@ import logging -from samtranslator.model.exceptions import InvalidResourceException, InvalidDocumentException from enum import Enum LOG = logging.getLogger(__name__) -class SamPlugins(object): - """ - Class providing support for arbitrary plugins that can extend core SAM translator in interesting ways. - Use this class to register plugins that get called when certain life cycle events happen in the translator. - Plugins work only on resources that are natively supported by SAM (ie. AWS::Serverless::* resources) - - Following Life Cycle Events are available: - - **Resource Level** - - before_transform_resource: Invoked before SAM translator processes a resource's properties. - - [Coming Soon] after_transform_resource - - **Template Level** - - before_transform_template - - after_transform_template - - When a life cycle event happens in the translator, this class will invoke the corresponding "hook" method on the - each of the registered plugins to process. Plugins are free to modify internal state of the template or resources - as they see fit. They can even raise an exception when the resource or template doesn't contain properties - of certain structure (Ex: Only PolicyTemplates are allowed in SAM template) - - ## Plugin Implementation - - ### Defining a plugin - A plugin is a subclass of `BasePlugin` that implements one or more methods capable of processing the life cycle - events. - These methods have a prefix `on_` followed by the name of the life cycle event. For example, to handle - `before_transform_resource` event, implement a method called `on_before_transform_resource`. We call these methods - as "hooks" which are methods capable of handling this event. - - ### Hook Methods - Arguments passed to the hook method is different for each life cycle event. Check out the hook methods in the - `BasePlugin` class for detailed description of the method signature - - ### Raising validation errors - Plugins must raise an `samtranslator.model.exception.InvalidResourceException` when the input SAM template does - not conform to the expectation - set by the plugin. SAM translator will convert this into a nice error message and display to the user. - """ - - def __init__(self, initial_plugins=None): - """ - Initialize the plugins class with an optional list of plugins - - :param BasePlugin or list initial_plugins: Single plugin or a List of plugins to initialize with - """ - self._plugins = [] - - if initial_plugins is None: - initial_plugins = [] - - if not isinstance(initial_plugins, list): - initial_plugins = [initial_plugins] - - for plugin in initial_plugins: - self.register(plugin) - - def register(self, plugin): - """ - Register a plugin. New plugins are added to the end of the plugins list. - - :param samtranslator.plugins.BasePlugin plugin: Instance/subclass of BasePlugin class that implements hooks - :raises ValueError: If plugin is not an instance of samtranslator.plugins.BasePlugin or if it is already - registered - :return: None - """ - - if not plugin or not isinstance(plugin, BasePlugin): - raise ValueError("Plugin must be implemented as a subclass of BasePlugin class") - - if self.is_registered(plugin.name): - raise ValueError("Plugin with name {} is already registered".format(plugin.name)) - - self._plugins.append(plugin) - - def is_registered(self, plugin_name): - """ - Checks if a plugin with given name is already registered - - :param plugin_name: Name of the plugin - :return: True if plugin with given name is already registered. False, otherwise - """ - - return plugin_name in [p.name for p in self._plugins] - - def _get(self, plugin_name): - """ - Retrieves the plugin with given name - - :param plugin_name: Name of the plugin to retrieve - :return samtranslator.plugins.BasePlugin: Returns the plugin object if found. None, otherwise - """ - - for p in self._plugins: - if p.name == plugin_name: - return p - - return None - - def act(self, event, *args, **kwargs): - """ - Act on the specific life cycle event. The action here is to invoke the hook function on all registered plugins. - *args and **kwargs will be passed directly to the plugin's hook functions - - :param samtranslator.plugins.LifeCycleEvents event: Event to act upon - :return: Nothing - :raises ValueError: If event is not a valid life cycle event - :raises NameError: If a plugin does not have the hook method defined - :raises Exception: Any exception that a plugin raises - """ - - if not isinstance(event, LifeCycleEvents): - raise ValueError("'event' must be an instance of LifeCycleEvents class") - - method_name = "on_" + event.name - - for plugin in self._plugins: - - if not hasattr(plugin, method_name): - raise NameError( - "'{}' method is not found in the plugin with name '{}'".format(method_name, plugin.name) - ) - - try: - getattr(plugin, method_name)(*args, **kwargs) - except (InvalidResourceException, InvalidDocumentException) as ex: - # Don't need to log these because they don't result in crashes - raise ex - except Exception as ex: - LOG.exception("Plugin '%s' raised an exception: %s", plugin.name, ex) - raise ex - - def __len__(self): - """ - Returns the number of plugins registered with this class - - :return integer: Number of plugins registered - """ - return len(self._plugins) - - class LifeCycleEvents(Enum): """ Enum of LifeCycleEvents diff --git a/samtranslator/plugins/application/serverless_app_plugin.py b/samtranslator/plugins/application/serverless_app_plugin.py index 5b1131e3b..32058291c 100644 --- a/samtranslator/plugins/application/serverless_app_plugin.py +++ b/samtranslator/plugins/application/serverless_app_plugin.py @@ -63,6 +63,7 @@ def __init__(self, sar_client=None, wait_for_template_active_status=False, valid self._wait_for_template_active_status = wait_for_template_active_status self._validate_only = validate_only self._parameters = parameters + self._total_wait_time = 0 # make sure the flag combination makes sense if self._validate_only is True and self._wait_for_template_active_status is True: @@ -118,11 +119,31 @@ def on_before_transform_template(self, template_dict): # Lazy initialization of the client- create it when it is needed if not self._sar_client: self._sar_client = boto3.client("serverlessrepo") - service_call(app_id, semver, key, logical_id) + self._make_service_call_with_retry(service_call, app_id, semver, key, logical_id) except InvalidResourceException as e: # Catch all InvalidResourceExceptions, raise those in the before_resource_transform target. self._applications[key] = e + def _make_service_call_with_retry(self, service_call, app_id, semver, key, logical_id): + call_succeeded = False + while self._total_wait_time < self.TEMPLATE_WAIT_TIMEOUT_SECONDS: + try: + service_call(app_id, semver, key, logical_id) + except ClientError as e: + error_code = e.response["Error"]["Code"] + if error_code == "TooManyRequestsException": + LOG.debug("SAR call timed out for application id {}".format(app_id)) + sleep_time = self._get_sleep_time_sec() + sleep(sleep_time) + self._total_wait_time += sleep_time + continue + else: + raise e + call_succeeded = True + break + if not call_succeeded: + raise InvalidResourceException(logical_id, "Failed to call SAR, timeout limit exceeded.") + def _replace_value(self, input_dict, key, intrinsic_resolvers): value = self._resolve_location_value(input_dict.get(key), intrinsic_resolvers) input_dict[key] = value @@ -307,8 +328,7 @@ def on_after_transform_template(self, template): if not self._wait_for_template_active_status or self._validate_only: return - start_time = time() - while (time() - start_time) < self.TEMPLATE_WAIT_TIMEOUT_SECONDS: + while self._total_wait_time < self.TEMPLATE_WAIT_TIMEOUT_SECONDS: # Check each resource to make sure it's active LOG.info("Checking resources in serverless application repo...") idx = 0 @@ -341,7 +361,9 @@ def on_after_transform_template(self, template): break # Sleep a little so we don't spam service calls - sleep(self._get_sleep_time_sec()) + sleep_time = self._get_sleep_time_sec() + sleep(sleep_time) + self._total_wait_time += sleep_time # Not all templates reached active status if len(self._in_progress_templates) != 0: diff --git a/samtranslator/plugins/sam_plugins.py b/samtranslator/plugins/sam_plugins.py new file mode 100644 index 000000000..c98e2cbb7 --- /dev/null +++ b/samtranslator/plugins/sam_plugins.py @@ -0,0 +1,146 @@ +import logging +from samtranslator.model.exceptions import InvalidResourceException, InvalidDocumentException +from samtranslator.plugins import BasePlugin, LifeCycleEvents + +LOG = logging.getLogger(__name__) + + +class SamPlugins(object): + """ + Class providing support for arbitrary plugins that can extend core SAM translator in interesting ways. + Use this class to register plugins that get called when certain life cycle events happen in the translator. + Plugins work only on resources that are natively supported by SAM (ie. AWS::Serverless::* resources) + + Following Life Cycle Events are available: + + **Resource Level** + - before_transform_resource: Invoked before SAM translator processes a resource's properties. + - [Coming Soon] after_transform_resource + + **Template Level** + - before_transform_template + - after_transform_template + + When a life cycle event happens in the translator, this class will invoke the corresponding "hook" method on the + each of the registered plugins to process. Plugins are free to modify internal state of the template or resources + as they see fit. They can even raise an exception when the resource or template doesn't contain properties + of certain structure (Ex: Only PolicyTemplates are allowed in SAM template) + + ## Plugin Implementation + + ### Defining a plugin + A plugin is a subclass of `BasePlugin` that implements one or more methods capable of processing the life cycle + events. + These methods have a prefix `on_` followed by the name of the life cycle event. For example, to handle + `before_transform_resource` event, implement a method called `on_before_transform_resource`. We call these methods + as "hooks" which are methods capable of handling this event. + + ### Hook Methods + Arguments passed to the hook method is different for each life cycle event. Check out the hook methods in the + `BasePlugin` class for detailed description of the method signature + + ### Raising validation errors + Plugins must raise an `samtranslator.model.exception.InvalidResourceException` when the input SAM template does + not conform to the expectation + set by the plugin. SAM translator will convert this into a nice error message and display to the user. + """ + + def __init__(self, initial_plugins=None): + """ + Initialize the plugins class with an optional list of plugins + + :param BasePlugin or list initial_plugins: Single plugin or a List of plugins to initialize with + """ + self._plugins = [] + + if initial_plugins is None: + initial_plugins = [] + + if not isinstance(initial_plugins, list): + initial_plugins = [initial_plugins] + + for plugin in initial_plugins: + self.register(plugin) + + def register(self, plugin): + """ + Register a plugin. New plugins are added to the end of the plugins list. + + :param samtranslator.plugins.BasePlugin plugin: Instance/subclass of BasePlugin class that implements hooks + :raises ValueError: If plugin is not an instance of samtranslator.plugins.BasePlugin or if it is already + registered + :return: None + """ + + if not plugin or not isinstance(plugin, BasePlugin): + raise ValueError("Plugin must be implemented as a subclass of BasePlugin class") + + if self.is_registered(plugin.name): + raise ValueError("Plugin with name {} is already registered".format(plugin.name)) + + self._plugins.append(plugin) + + def is_registered(self, plugin_name): + """ + Checks if a plugin with given name is already registered + + :param plugin_name: Name of the plugin + :return: True if plugin with given name is already registered. False, otherwise + """ + + return plugin_name in [p.name for p in self._plugins] + + def _get(self, plugin_name): + """ + Retrieves the plugin with given name + + :param plugin_name: Name of the plugin to retrieve + :return samtranslator.plugins.BasePlugin: Returns the plugin object if found. None, otherwise + """ + + for p in self._plugins: + if p.name == plugin_name: + return p + + return None + + def act(self, event, *args, **kwargs): + """ + Act on the specific life cycle event. The action here is to invoke the hook function on all registered plugins. + *args and **kwargs will be passed directly to the plugin's hook functions + + :param samtranslator.plugins.LifeCycleEvents event: Event to act upon + :return: Nothing + :raises ValueError: If event is not a valid life cycle event + :raises NameError: If a plugin does not have the hook method defined + :raises Exception: Any exception that a plugin raises + """ + + if not isinstance(event, LifeCycleEvents): + raise ValueError("'event' must be an instance of LifeCycleEvents class") + + method_name = "on_" + event.name + + for plugin in self._plugins: + + if not hasattr(plugin, method_name): + raise NameError( + "'{}' method is not found in the plugin with name '{}'".format(method_name, plugin.name) + ) + + try: + getattr(plugin, method_name)(*args, **kwargs) + except (InvalidResourceException, InvalidDocumentException) as ex: + # Don't need to log these because they don't result in crashes + raise ex + except Exception as ex: + LOG.exception("Plugin '%s' raised an exception: %s", plugin.name, ex) + raise ex + + def __len__(self): + """ + Returns the number of plugins registered with this class + + :return integer: Number of plugins registered + """ + return len(self._plugins) diff --git a/samtranslator/translator/translator.py b/samtranslator/translator/translator.py index 5a4fe388f..1568091d9 100644 --- a/samtranslator/translator/translator.py +++ b/samtranslator/translator/translator.py @@ -23,7 +23,7 @@ from samtranslator.plugins.api.default_definition_body_plugin import DefaultDefinitionBodyPlugin from samtranslator.plugins.application.serverless_app_plugin import ServerlessAppPlugin from samtranslator.plugins import LifeCycleEvents -from samtranslator.plugins import SamPlugins +from samtranslator.plugins.sam_plugins import SamPlugins from samtranslator.plugins.globals.globals_plugin import GlobalsPlugin from samtranslator.plugins.policies.policy_templates_plugin import PolicyTemplatesForResourcePlugin from samtranslator.policy_template_processor.processor import PolicyTemplatesProcessor diff --git a/tests/plugins/application/test_serverless_app_plugin.py b/tests/plugins/application/test_serverless_app_plugin.py index 5b56846db..cfe67160c 100644 --- a/tests/plugins/application/test_serverless_app_plugin.py +++ b/tests/plugins/application/test_serverless_app_plugin.py @@ -220,10 +220,120 @@ def test_resolve_intrinsics(self): self.assertEqual("value1", output) + @patch("samtranslator.plugins.application.serverless_app_plugin.SamTemplate") + def test_sar_throttling_doesnt_stop_processing(self, SamTemplateMock): + client = Mock() + client.create_cloud_formation_template = Mock() + client.create_cloud_formation_template.side_effect = ClientError( + {"Error": {"Code": "TooManyRequestsException"}}, "CreateCloudFormationTemplate" + ) + + app_resources = [ + ("id1", ApplicationResource(app_id="id1", semver="1.0.0", location=True)), + ] + + sam_template = Mock() + SamTemplateMock.return_value = sam_template + sam_template.iterate = Mock() + sam_template.iterate.return_value = app_resources + + self.plugin = ServerlessAppPlugin(sar_client=client) + self.plugin._can_process_application = Mock() + self.plugin._can_process_application.return_value = True + self.plugin._get_sleep_time_sec = Mock() + self.plugin._get_sleep_time_sec.return_value = 0.02 + self.plugin.TEMPLATE_WAIT_TIMEOUT_SECONDS = 1.0 + + self.plugin.on_before_transform_template({}) + self.assertEqual( + self.plugin._applications.get(("id1", "1.0.0")).message, + "Resource with id [id1] is invalid. Failed to call SAR, timeout limit exceeded.", + ) + # confirm we had at least two attempts to call SAR and that we executed a sleep + self.assertGreater(client.create_cloud_formation_template.call_count, 1) + self.assertGreaterEqual(self.plugin._get_sleep_time_sec.call_count, 1) + + @patch("samtranslator.plugins.application.serverless_app_plugin.SamTemplate") + def test_unexpected_sar_error_stops_processing(self, SamTemplateMock): + template_dict = {"a": "b"} + app_resources = [ + ("id1", ApplicationResource(app_id="id1", semver="1.0.0", location=True)), + ] + + sam_template = Mock() + SamTemplateMock.return_value = sam_template + sam_template.iterate = Mock() + sam_template.iterate.return_value = app_resources + + client = Mock() + client.create_cloud_formation_template.side_effect = ClientError( + {"Error": {"Code": "BadBadError"}}, "CreateCloudFormationTemplate" + ) + self.plugin = ServerlessAppPlugin(sar_client=client) + self.plugin._can_process_application = Mock() + self.plugin._can_process_application.return_value = True + + with self.assertRaises(ClientError): + self.plugin.on_before_transform_template(template_dict) + + @patch("samtranslator.plugins.application.serverless_app_plugin.SamTemplate") + def test_sar_success_one_app(self, SamTemplateMock): + template_dict = {"a": "b"} + app_resources = [ + ("id1", ApplicationResource(app_id="id1", semver="1.0.0", location=True)), + ] + + sam_template = Mock() + SamTemplateMock.return_value = sam_template + sam_template.iterate = Mock() + sam_template.iterate.return_value = app_resources + + client = Mock() + client.create_cloud_formation_template = Mock() + client.create_cloud_formation_template.return_value = {"TemplateUrl": "/URL", "Status": STATUS_ACTIVE} + self.plugin = ServerlessAppPlugin(sar_client=client) + self.plugin._can_process_application = Mock() + self.plugin._can_process_application.return_value = True + self.plugin.on_before_transform_template(template_dict) + + self.assertEqual(client.create_cloud_formation_template.call_count, 1) + + @patch("samtranslator.plugins.application.serverless_app_plugin.SamTemplate") + def test_sleep_between_sar_checks(self, SamTemplateMock): + template_dict = {"a": "b"} + client = Mock() + + app_resources = [ + ("id1", ApplicationResource(app_id="id1", semver="1.0.0", location=True)), + ] + + sam_template = Mock() + SamTemplateMock.return_value = sam_template + sam_template.iterate = Mock() + sam_template.iterate.return_value = app_resources + client.create_cloud_formation_template = Mock() + client.create_cloud_formation_template.side_effect = [ + ClientError({"Error": {"Code": "TooManyRequestsException"}}, "CreateCloudFormationTemplate"), + {"TemplateUrl": "/URL", "Status": STATUS_ACTIVE}, + ] + self.plugin._can_process_application = Mock() + self.plugin._can_process_application.return_value = True + self.plugin = ServerlessAppPlugin(sar_client=client, wait_for_template_active_status=True, validate_only=False) + self.plugin._get_sleep_time_sec = Mock() + self.plugin._get_sleep_time_sec.return_value = 0.001 + self.plugin.on_before_transform_template(template_dict) + # should have exactly two calls to SAR + self.assertEqual(client.create_cloud_formation_template.call_count, 2) + self.assertEqual(self.plugin._get_sleep_time_sec.call_count, 1) # make sure we slept once + class ApplicationResource(object): - def __init__(self, app_id="app_id", semver="1.3.5"): - self.properties = {"ApplicationId": app_id, "SemanticVersion": semver} + def __init__(self, app_id="app_id", semver="1.3.5", location=None): + self.properties = ( + {"ApplicationId": app_id, "SemanticVersion": semver} + if not location + else {"Location": {"ApplicationId": app_id, "SemanticVersion": semver}} + ) # class TestServerlessAppPlugin_on_before_transform_resource(TestCase): @@ -328,3 +438,42 @@ def test_sleep_between_sar_checks(self): # should have exactly two calls to SAR self.assertEqual(client.get_cloud_formation_template.call_count, 2) self.assertEqual(plugin._get_sleep_time_sec.call_count, 1) # make sure we slept once + + +class TestServerlessAppPlugin_on_before_and_on_after_transform_template(TestCase): + @patch("samtranslator.plugins.application.serverless_app_plugin.SamTemplate") + def test_time_limit_exceeds_between_combined_sar_calls(self, SamTemplateMock): + template_dict = {"a": "b"} + app_resources = [ + ("id1", ApplicationResource(app_id="id1", semver="1.0.0", location=True)), + ] + + sam_template = Mock() + SamTemplateMock.return_value = sam_template + sam_template.iterate = Mock() + sam_template.iterate.return_value = app_resources + + client = Mock() + client.get_cloud_formation_template = Mock() + client.get_cloud_formation_template.side_effect = [ + ClientError({"Error": {"Code": "TooManyRequestsException"}}, "GetCloudFormationTemplate"), + {"Status": STATUS_ACTIVE}, + ] + client.create_cloud_formation_template = Mock() + client.create_cloud_formation_template.side_effect = [ + ClientError({"Error": {"Code": "TooManyRequestsException"}}, "CreateCloudFormationTemplate"), + {"TemplateUrl": "/URL", "Status": STATUS_ACTIVE}, + ] + plugin = ServerlessAppPlugin(sar_client=client, wait_for_template_active_status=True, validate_only=False) + plugin._get_sleep_time_sec = Mock() + plugin._get_sleep_time_sec.return_value = 0.04 + plugin._in_progress_templates = [("appid", "template"), ("appid2", "template2")] + plugin.TEMPLATE_WAIT_TIMEOUT_SECONDS = 0.08 + + plugin.on_before_transform_template(template_dict) + with self.assertRaises(InvalidResourceException): + plugin.on_after_transform_template(template_dict) + # confirm we had at least two attempts to call SAR and that we executed a sleep + self.assertEqual(client.get_cloud_formation_template.call_count, 1) + self.assertEqual(client.create_cloud_formation_template.call_count, 2) + self.assertGreaterEqual(plugin._get_sleep_time_sec.call_count, 2) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 79f7080d9..344ead6c1 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,5 +1,6 @@ from enum import Enum -from samtranslator.plugins import SamPlugins, BasePlugin, LifeCycleEvents +from samtranslator.plugins import BasePlugin, LifeCycleEvents +from samtranslator.plugins.sam_plugins import SamPlugins from unittest import TestCase from unittest.mock import Mock, patch, call