fix: Add retries to SAR service calls on throttle

samtranslator/plugins/__init__.py

@@ -1,152 +1,10 @@
 import logging
 
-from samtranslator.model.exceptions import InvalidResourceException, InvalidDocumentException
 from enum import Enum
 
 LOG = logging.getLogger(__name__)
 
 
-class SamPlugins(object):
-    """
-    Class providing support for arbitrary plugins that can extend core SAM translator in interesting ways.
-    Use this class to register plugins that get called when certain life cycle events happen in the translator.
-    Plugins work only on resources that are natively supported by SAM (ie. AWS::Serverless::* resources)
-
-    Following Life Cycle Events are available:
-
-    **Resource Level**
-    - before_transform_resource: Invoked before SAM translator processes a resource's properties.
-    - [Coming Soon] after_transform_resource
-
-    **Template Level**
-    - before_transform_template
-    - after_transform_template
-
-    When a life cycle event happens in the translator, this class will invoke the corresponding "hook" method on the
-    each of the registered plugins to process. Plugins are free to modify internal state of the template or resources
-    as they see fit. They can even raise an exception when the resource or template doesn't contain properties
-    of certain structure (Ex: Only PolicyTemplates are allowed in SAM template)
-
-    ## Plugin Implementation
-
-    ### Defining a plugin
-    A plugin is a subclass of `BasePlugin` that implements one or more methods capable of processing the life cycle
-    events.
-    These methods have a prefix `on_` followed by the name of the life cycle event. For example, to  handle
-    `before_transform_resource` event, implement a method called `on_before_transform_resource`. We call these methods
-    as "hooks" which are methods capable of handling this event.
-
-    ### Hook Methods
-    Arguments passed to the hook method is different for each life cycle event. Check out the hook methods in the
-    `BasePlugin` class for detailed description of the method signature
-
-    ### Raising validation errors
-    Plugins must raise an `samtranslator.model.exception.InvalidResourceException` when the input SAM template does
-    not conform to the expectation
-    set by the plugin. SAM translator will convert this into a nice error message and display to the user.
-    """
-
-    def __init__(self, initial_plugins=None):
-        """
-        Initialize the plugins class with an optional list of plugins
-
-        :param BasePlugin or list initial_plugins: Single plugin or a List of plugins to initialize with
-        """
-        self._plugins = []
-
-        if initial_plugins is None:
-            initial_plugins = []
-
-        if not isinstance(initial_plugins, list):
-            initial_plugins = [initial_plugins]
-
-        for plugin in initial_plugins:
-            self.register(plugin)
-
-    def register(self, plugin):
-        """
-        Register a plugin. New plugins are added to the end of the plugins list.
-
-        :param samtranslator.plugins.BasePlugin plugin: Instance/subclass of BasePlugin class that implements hooks
-        :raises ValueError: If plugin is not an instance of samtranslator.plugins.BasePlugin or if it is already
-            registered
-        :return: None
-        """
-
-        if not plugin or not isinstance(plugin, BasePlugin):
-            raise ValueError("Plugin must be implemented as a subclass of BasePlugin class")
-
-        if self.is_registered(plugin.name):
-            raise ValueError("Plugin with name {} is already registered".format(plugin.name))
-
-        self._plugins.append(plugin)
-
-    def is_registered(self, plugin_name):
-        """
-        Checks if a plugin with given name is already registered
-
-        :param plugin_name: Name of the plugin
-        :return: True if plugin with given name is already registered. False, otherwise
-        """
-
-        return plugin_name in [p.name for p in self._plugins]
-
-    def _get(self, plugin_name):
-        """
-        Retrieves the plugin with given name
-
-        :param plugin_name: Name of the plugin to retrieve
-        :return samtranslator.plugins.BasePlugin: Returns the plugin object if found. None, otherwise
-        """
-
-        for p in self._plugins:
-            if p.name == plugin_name:
-                return p
-
-        return None
-
-    def act(self, event, *args, **kwargs):
-        """
-        Act on the specific life cycle event. The action here is to invoke the hook function on all registered plugins.
-        *args and **kwargs will be passed directly to the plugin's hook functions
-
-        :param samtranslator.plugins.LifeCycleEvents event: Event to act upon
-        :return: Nothing
-        :raises ValueError: If event is not a valid life cycle event
-        :raises NameError: If a plugin does not have the hook method defined
-        :raises Exception: Any exception that a plugin raises
-        """
-
-        if not isinstance(event, LifeCycleEvents):
-            raise ValueError("'event' must be an instance of LifeCycleEvents class")
-
-        method_name = "on_" + event.name
-
-        for plugin in self._plugins:
-
-            if not hasattr(plugin, method_name):
-                raise NameError(
-                    "'{}' method is not found in the plugin with name '{}'".format(method_name, plugin.name)
-                )
-
-            try:
-                getattr(plugin, method_name)(*args, **kwargs)
-            except (InvalidResourceException, InvalidDocumentException) as ex:
-                # Don't need to log these because they don't result in crashes
-                raise ex
-            except Exception as ex:
-                LOG.exception("Plugin '%s' raised an exception: %s", plugin.name, ex)
-                raise ex
-
-    def __len__(self):
-        """
-        Returns the number of plugins registered with this class
-
-        :return integer: Number of plugins registered
-        """
-        return len(self._plugins)
-
-
 class LifeCycleEvents(Enum):
     """
     Enum of LifeCycleEvents

samtranslator/plugins/application/serverless_app_plugin.py

@@ -63,6 +63,7 @@ def __init__(self, sar_client=None, wait_for_template_active_status=False, valid
         self._wait_for_template_active_status = wait_for_template_active_status
         self._validate_only = validate_only
         self._parameters = parameters
+        self._total_wait_time = 0
 
         # make sure the flag combination makes sense
         if self._validate_only is True and self._wait_for_template_active_status is True:
@@ -118,11 +119,31 @@ def on_before_transform_template(self, template_dict):
                     # Lazy initialization of the client- create it when it is needed
                     if not self._sar_client:
                         self._sar_client = boto3.client("serverlessrepo")
-                    service_call(app_id, semver, key, logical_id)
+                    self._make_service_call_with_retry(service_call, app_id, semver, key, logical_id)
                 except InvalidResourceException as e:
                     # Catch all InvalidResourceExceptions, raise those in the before_resource_transform target.
                     self._applications[key] = e
 
+    def _make_service_call_with_retry(self, service_call, app_id, semver, key, logical_id):
+        call_succeeded = False
+        while self._total_wait_time < self.TEMPLATE_WAIT_TIMEOUT_SECONDS:
+            try:
+                service_call(app_id, semver, key, logical_id)
+            except ClientError as e:
+                error_code = e.response["Error"]["Code"]
+                if error_code == "TooManyRequestsException":
+                    LOG.debug("SAR call timed out for application id {}".format(app_id))
+                    sleep_time = self._get_sleep_time_sec()
+                    sleep(sleep_time)
+                    self._total_wait_time += sleep_time
+                    continue
+                else:
+                    raise e
+            call_succeeded = True
+            break
+        if not call_succeeded:
+            raise InvalidResourceException(logical_id, "Failed to call SAR, timeout limit exceeded.")
+
     def _replace_value(self, input_dict, key, intrinsic_resolvers):
         value = self._resolve_location_value(input_dict.get(key), intrinsic_resolvers)
         input_dict[key] = value
@@ -307,8 +328,7 @@ def on_after_transform_template(self, template):
         if not self._wait_for_template_active_status or self._validate_only:
             return
 
-        start_time = time()
-        while (time() - start_time) < self.TEMPLATE_WAIT_TIMEOUT_SECONDS:
+        while self._total_wait_time < self.TEMPLATE_WAIT_TIMEOUT_SECONDS:
             # Check each resource to make sure it's active
             LOG.info("Checking resources in serverless application repo...")
             idx = 0
@@ -341,7 +361,9 @@ def on_after_transform_template(self, template):
                 break
 
             # Sleep a little so we don't spam service calls
-            sleep(self._get_sleep_time_sec())
+            sleep_time = self._get_sleep_time_sec()
+            sleep(sleep_time)
+            self._total_wait_time += sleep_time
 
         # Not all templates reached active status
         if len(self._in_progress_templates) != 0:

samtranslator/plugins/sam_plugins.py

@@ -0,0 +1,146 @@
+import logging
+from samtranslator.model.exceptions import InvalidResourceException, InvalidDocumentException
+from samtranslator.plugins import BasePlugin, LifeCycleEvents
+
+LOG = logging.getLogger(__name__)
+
+
+class SamPlugins(object):
+    """
+    Class providing support for arbitrary plugins that can extend core SAM translator in interesting ways.
+    Use this class to register plugins that get called when certain life cycle events happen in the translator.
+    Plugins work only on resources that are natively supported by SAM (ie. AWS::Serverless::* resources)
+
+    Following Life Cycle Events are available:
+
+    **Resource Level**
+    - before_transform_resource: Invoked before SAM translator processes a resource's properties.
+    - [Coming Soon] after_transform_resource
+
+    **Template Level**
+    - before_transform_template
+    - after_transform_template
+
+    When a life cycle event happens in the translator, this class will invoke the corresponding "hook" method on the
+    each of the registered plugins to process. Plugins are free to modify internal state of the template or resources
+    as they see fit. They can even raise an exception when the resource or template doesn't contain properties
+    of certain structure (Ex: Only PolicyTemplates are allowed in SAM template)
+
+    ## Plugin Implementation
+
+    ### Defining a plugin
+    A plugin is a subclass of `BasePlugin` that implements one or more methods capable of processing the life cycle
+    events.
+    These methods have a prefix `on_` followed by the name of the life cycle event. For example, to  handle
+    `before_transform_resource` event, implement a method called `on_before_transform_resource`. We call these methods
+    as "hooks" which are methods capable of handling this event.
+
+    ### Hook Methods
+    Arguments passed to the hook method is different for each life cycle event. Check out the hook methods in the
+    `BasePlugin` class for detailed description of the method signature
+
+    ### Raising validation errors
+    Plugins must raise an `samtranslator.model.exception.InvalidResourceException` when the input SAM template does
+    not conform to the expectation
+    set by the plugin. SAM translator will convert this into a nice error message and display to the user.
+    """
+
+    def __init__(self, initial_plugins=None):
+        """
+        Initialize the plugins class with an optional list of plugins
+
+        :param BasePlugin or list initial_plugins: Single plugin or a List of plugins to initialize with
+        """
+        self._plugins = []
+
+        if initial_plugins is None:
+            initial_plugins = []
+
+        if not isinstance(initial_plugins, list):
+            initial_plugins = [initial_plugins]
+
+        for plugin in initial_plugins:
+            self.register(plugin)
+
+    def register(self, plugin):
+        """
+        Register a plugin. New plugins are added to the end of the plugins list.
+
+        :param samtranslator.plugins.BasePlugin plugin: Instance/subclass of BasePlugin class that implements hooks
+        :raises ValueError: If plugin is not an instance of samtranslator.plugins.BasePlugin or if it is already
+            registered
+        :return: None
+        """
+
+        if not plugin or not isinstance(plugin, BasePlugin):
+            raise ValueError("Plugin must be implemented as a subclass of BasePlugin class")
+
+        if self.is_registered(plugin.name):
+            raise ValueError("Plugin with name {} is already registered".format(plugin.name))
+
+        self._plugins.append(plugin)
+
+    def is_registered(self, plugin_name):
+        """
+        Checks if a plugin with given name is already registered
+
+        :param plugin_name: Name of the plugin
+        :return: True if plugin with given name is already registered. False, otherwise
+        """
+
+        return plugin_name in [p.name for p in self._plugins]
+
+    def _get(self, plugin_name):
+        """
+        Retrieves the plugin with given name
+
+        :param plugin_name: Name of the plugin to retrieve
+        :return samtranslator.plugins.BasePlugin: Returns the plugin object if found. None, otherwise
+        """
+
+        for p in self._plugins:
+            if p.name == plugin_name:
+                return p
+
+        return None
+
+    def act(self, event, *args, **kwargs):
+        """
+        Act on the specific life cycle event. The action here is to invoke the hook function on all registered plugins.
+        *args and **kwargs will be passed directly to the plugin's hook functions
+
+        :param samtranslator.plugins.LifeCycleEvents event: Event to act upon
+        :return: Nothing
+        :raises ValueError: If event is not a valid life cycle event
+        :raises NameError: If a plugin does not have the hook method defined
+        :raises Exception: Any exception that a plugin raises
+        """
+
+        if not isinstance(event, LifeCycleEvents):
+            raise ValueError("'event' must be an instance of LifeCycleEvents class")
+
+        method_name = "on_" + event.name
+
+        for plugin in self._plugins:
+
+            if not hasattr(plugin, method_name):
+                raise NameError(
+                    "'{}' method is not found in the plugin with name '{}'".format(method_name, plugin.name)
+                )
+
+            try:
+                getattr(plugin, method_name)(*args, **kwargs)
+            except (InvalidResourceException, InvalidDocumentException) as ex:
+                # Don't need to log these because they don't result in crashes
+                raise ex
+            except Exception as ex:
+                LOG.exception("Plugin '%s' raised an exception: %s", plugin.name, ex)
+                raise ex
+
+    def __len__(self):
+        """
+        Returns the number of plugins registered with this class
+
+        :return integer: Number of plugins registered
+        """
+        return len(self._plugins)

samtranslator/translator/translator.py

@@ -23,7 +23,7 @@
 from samtranslator.plugins.api.default_definition_body_plugin import DefaultDefinitionBodyPlugin
 from samtranslator.plugins.application.serverless_app_plugin import ServerlessAppPlugin
 from samtranslator.plugins import LifeCycleEvents
-from samtranslator.plugins import SamPlugins
+from samtranslator.plugins.sam_plugins import SamPlugins
 from samtranslator.plugins.globals.globals_plugin import GlobalsPlugin
 from samtranslator.plugins.policies.policy_templates_plugin import PolicyTemplatesForResourcePlugin
 from samtranslator.policy_template_processor.processor import PolicyTemplatesProcessor