Skip to content

Commit a3c40bb

Browse files
authored
fix: Add retries to SAR service calls on throttle (#2352)
* Add retries to SAR service calls on throttle * Update variable names for clarity * Use cumulative timeout counter
1 parent 66f6d3b commit a3c40bb

File tree

6 files changed

+326
-150
lines changed

6 files changed

+326
-150
lines changed

samtranslator/plugins/__init__.py

Lines changed: 0 additions & 142 deletions
Original file line numberDiff line numberDiff line change
@@ -1,152 +1,10 @@
11
import logging
22

3-
from samtranslator.model.exceptions import InvalidResourceException, InvalidDocumentException
43
from enum import Enum
54

65
LOG = logging.getLogger(__name__)
76

87

9-
class SamPlugins(object):
10-
"""
11-
Class providing support for arbitrary plugins that can extend core SAM translator in interesting ways.
12-
Use this class to register plugins that get called when certain life cycle events happen in the translator.
13-
Plugins work only on resources that are natively supported by SAM (ie. AWS::Serverless::* resources)
14-
15-
Following Life Cycle Events are available:
16-
17-
**Resource Level**
18-
- before_transform_resource: Invoked before SAM translator processes a resource's properties.
19-
- [Coming Soon] after_transform_resource
20-
21-
**Template Level**
22-
- before_transform_template
23-
- after_transform_template
24-
25-
When a life cycle event happens in the translator, this class will invoke the corresponding "hook" method on the
26-
each of the registered plugins to process. Plugins are free to modify internal state of the template or resources
27-
as they see fit. They can even raise an exception when the resource or template doesn't contain properties
28-
of certain structure (Ex: Only PolicyTemplates are allowed in SAM template)
29-
30-
## Plugin Implementation
31-
32-
### Defining a plugin
33-
A plugin is a subclass of `BasePlugin` that implements one or more methods capable of processing the life cycle
34-
events.
35-
These methods have a prefix `on_` followed by the name of the life cycle event. For example, to handle
36-
`before_transform_resource` event, implement a method called `on_before_transform_resource`. We call these methods
37-
as "hooks" which are methods capable of handling this event.
38-
39-
### Hook Methods
40-
Arguments passed to the hook method is different for each life cycle event. Check out the hook methods in the
41-
`BasePlugin` class for detailed description of the method signature
42-
43-
### Raising validation errors
44-
Plugins must raise an `samtranslator.model.exception.InvalidResourceException` when the input SAM template does
45-
not conform to the expectation
46-
set by the plugin. SAM translator will convert this into a nice error message and display to the user.
47-
"""
48-
49-
def __init__(self, initial_plugins=None):
50-
"""
51-
Initialize the plugins class with an optional list of plugins
52-
53-
:param BasePlugin or list initial_plugins: Single plugin or a List of plugins to initialize with
54-
"""
55-
self._plugins = []
56-
57-
if initial_plugins is None:
58-
initial_plugins = []
59-
60-
if not isinstance(initial_plugins, list):
61-
initial_plugins = [initial_plugins]
62-
63-
for plugin in initial_plugins:
64-
self.register(plugin)
65-
66-
def register(self, plugin):
67-
"""
68-
Register a plugin. New plugins are added to the end of the plugins list.
69-
70-
:param samtranslator.plugins.BasePlugin plugin: Instance/subclass of BasePlugin class that implements hooks
71-
:raises ValueError: If plugin is not an instance of samtranslator.plugins.BasePlugin or if it is already
72-
registered
73-
:return: None
74-
"""
75-
76-
if not plugin or not isinstance(plugin, BasePlugin):
77-
raise ValueError("Plugin must be implemented as a subclass of BasePlugin class")
78-
79-
if self.is_registered(plugin.name):
80-
raise ValueError("Plugin with name {} is already registered".format(plugin.name))
81-
82-
self._plugins.append(plugin)
83-
84-
def is_registered(self, plugin_name):
85-
"""
86-
Checks if a plugin with given name is already registered
87-
88-
:param plugin_name: Name of the plugin
89-
:return: True if plugin with given name is already registered. False, otherwise
90-
"""
91-
92-
return plugin_name in [p.name for p in self._plugins]
93-
94-
def _get(self, plugin_name):
95-
"""
96-
Retrieves the plugin with given name
97-
98-
:param plugin_name: Name of the plugin to retrieve
99-
:return samtranslator.plugins.BasePlugin: Returns the plugin object if found. None, otherwise
100-
"""
101-
102-
for p in self._plugins:
103-
if p.name == plugin_name:
104-
return p
105-
106-
return None
107-
108-
def act(self, event, *args, **kwargs):
109-
"""
110-
Act on the specific life cycle event. The action here is to invoke the hook function on all registered plugins.
111-
*args and **kwargs will be passed directly to the plugin's hook functions
112-
113-
:param samtranslator.plugins.LifeCycleEvents event: Event to act upon
114-
:return: Nothing
115-
:raises ValueError: If event is not a valid life cycle event
116-
:raises NameError: If a plugin does not have the hook method defined
117-
:raises Exception: Any exception that a plugin raises
118-
"""
119-
120-
if not isinstance(event, LifeCycleEvents):
121-
raise ValueError("'event' must be an instance of LifeCycleEvents class")
122-
123-
method_name = "on_" + event.name
124-
125-
for plugin in self._plugins:
126-
127-
if not hasattr(plugin, method_name):
128-
raise NameError(
129-
"'{}' method is not found in the plugin with name '{}'".format(method_name, plugin.name)
130-
)
131-
132-
try:
133-
getattr(plugin, method_name)(*args, **kwargs)
134-
except (InvalidResourceException, InvalidDocumentException) as ex:
135-
# Don't need to log these because they don't result in crashes
136-
raise ex
137-
except Exception as ex:
138-
LOG.exception("Plugin '%s' raised an exception: %s", plugin.name, ex)
139-
raise ex
140-
141-
def __len__(self):
142-
"""
143-
Returns the number of plugins registered with this class
144-
145-
:return integer: Number of plugins registered
146-
"""
147-
return len(self._plugins)
148-
149-
1508
class LifeCycleEvents(Enum):
1519
"""
15210
Enum of LifeCycleEvents

samtranslator/plugins/application/serverless_app_plugin.py

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ def __init__(self, sar_client=None, wait_for_template_active_status=False, valid
6363
self._wait_for_template_active_status = wait_for_template_active_status
6464
self._validate_only = validate_only
6565
self._parameters = parameters
66+
self._total_wait_time = 0
6667

6768
# make sure the flag combination makes sense
6869
if self._validate_only is True and self._wait_for_template_active_status is True:
@@ -118,11 +119,31 @@ def on_before_transform_template(self, template_dict):
118119
# Lazy initialization of the client- create it when it is needed
119120
if not self._sar_client:
120121
self._sar_client = boto3.client("serverlessrepo")
121-
service_call(app_id, semver, key, logical_id)
122+
self._make_service_call_with_retry(service_call, app_id, semver, key, logical_id)
122123
except InvalidResourceException as e:
123124
# Catch all InvalidResourceExceptions, raise those in the before_resource_transform target.
124125
self._applications[key] = e
125126

127+
def _make_service_call_with_retry(self, service_call, app_id, semver, key, logical_id):
128+
call_succeeded = False
129+
while self._total_wait_time < self.TEMPLATE_WAIT_TIMEOUT_SECONDS:
130+
try:
131+
service_call(app_id, semver, key, logical_id)
132+
except ClientError as e:
133+
error_code = e.response["Error"]["Code"]
134+
if error_code == "TooManyRequestsException":
135+
LOG.debug("SAR call timed out for application id {}".format(app_id))
136+
sleep_time = self._get_sleep_time_sec()
137+
sleep(sleep_time)
138+
self._total_wait_time += sleep_time
139+
continue
140+
else:
141+
raise e
142+
call_succeeded = True
143+
break
144+
if not call_succeeded:
145+
raise InvalidResourceException(logical_id, "Failed to call SAR, timeout limit exceeded.")
146+
126147
def _replace_value(self, input_dict, key, intrinsic_resolvers):
127148
value = self._resolve_location_value(input_dict.get(key), intrinsic_resolvers)
128149
input_dict[key] = value
@@ -307,8 +328,7 @@ def on_after_transform_template(self, template):
307328
if not self._wait_for_template_active_status or self._validate_only:
308329
return
309330

310-
start_time = time()
311-
while (time() - start_time) < self.TEMPLATE_WAIT_TIMEOUT_SECONDS:
331+
while self._total_wait_time < self.TEMPLATE_WAIT_TIMEOUT_SECONDS:
312332
# Check each resource to make sure it's active
313333
LOG.info("Checking resources in serverless application repo...")
314334
idx = 0
@@ -341,7 +361,9 @@ def on_after_transform_template(self, template):
341361
break
342362

343363
# Sleep a little so we don't spam service calls
344-
sleep(self._get_sleep_time_sec())
364+
sleep_time = self._get_sleep_time_sec()
365+
sleep(sleep_time)
366+
self._total_wait_time += sleep_time
345367

346368
# Not all templates reached active status
347369
if len(self._in_progress_templates) != 0:
Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
import logging
2+
from samtranslator.model.exceptions import InvalidResourceException, InvalidDocumentException
3+
from samtranslator.plugins import BasePlugin, LifeCycleEvents
4+
5+
LOG = logging.getLogger(__name__)
6+
7+
8+
class SamPlugins(object):
9+
"""
10+
Class providing support for arbitrary plugins that can extend core SAM translator in interesting ways.
11+
Use this class to register plugins that get called when certain life cycle events happen in the translator.
12+
Plugins work only on resources that are natively supported by SAM (ie. AWS::Serverless::* resources)
13+
14+
Following Life Cycle Events are available:
15+
16+
**Resource Level**
17+
- before_transform_resource: Invoked before SAM translator processes a resource's properties.
18+
- [Coming Soon] after_transform_resource
19+
20+
**Template Level**
21+
- before_transform_template
22+
- after_transform_template
23+
24+
When a life cycle event happens in the translator, this class will invoke the corresponding "hook" method on the
25+
each of the registered plugins to process. Plugins are free to modify internal state of the template or resources
26+
as they see fit. They can even raise an exception when the resource or template doesn't contain properties
27+
of certain structure (Ex: Only PolicyTemplates are allowed in SAM template)
28+
29+
## Plugin Implementation
30+
31+
### Defining a plugin
32+
A plugin is a subclass of `BasePlugin` that implements one or more methods capable of processing the life cycle
33+
events.
34+
These methods have a prefix `on_` followed by the name of the life cycle event. For example, to handle
35+
`before_transform_resource` event, implement a method called `on_before_transform_resource`. We call these methods
36+
as "hooks" which are methods capable of handling this event.
37+
38+
### Hook Methods
39+
Arguments passed to the hook method is different for each life cycle event. Check out the hook methods in the
40+
`BasePlugin` class for detailed description of the method signature
41+
42+
### Raising validation errors
43+
Plugins must raise an `samtranslator.model.exception.InvalidResourceException` when the input SAM template does
44+
not conform to the expectation
45+
set by the plugin. SAM translator will convert this into a nice error message and display to the user.
46+
"""
47+
48+
def __init__(self, initial_plugins=None):
49+
"""
50+
Initialize the plugins class with an optional list of plugins
51+
52+
:param BasePlugin or list initial_plugins: Single plugin or a List of plugins to initialize with
53+
"""
54+
self._plugins = []
55+
56+
if initial_plugins is None:
57+
initial_plugins = []
58+
59+
if not isinstance(initial_plugins, list):
60+
initial_plugins = [initial_plugins]
61+
62+
for plugin in initial_plugins:
63+
self.register(plugin)
64+
65+
def register(self, plugin):
66+
"""
67+
Register a plugin. New plugins are added to the end of the plugins list.
68+
69+
:param samtranslator.plugins.BasePlugin plugin: Instance/subclass of BasePlugin class that implements hooks
70+
:raises ValueError: If plugin is not an instance of samtranslator.plugins.BasePlugin or if it is already
71+
registered
72+
:return: None
73+
"""
74+
75+
if not plugin or not isinstance(plugin, BasePlugin):
76+
raise ValueError("Plugin must be implemented as a subclass of BasePlugin class")
77+
78+
if self.is_registered(plugin.name):
79+
raise ValueError("Plugin with name {} is already registered".format(plugin.name))
80+
81+
self._plugins.append(plugin)
82+
83+
def is_registered(self, plugin_name):
84+
"""
85+
Checks if a plugin with given name is already registered
86+
87+
:param plugin_name: Name of the plugin
88+
:return: True if plugin with given name is already registered. False, otherwise
89+
"""
90+
91+
return plugin_name in [p.name for p in self._plugins]
92+
93+
def _get(self, plugin_name):
94+
"""
95+
Retrieves the plugin with given name
96+
97+
:param plugin_name: Name of the plugin to retrieve
98+
:return samtranslator.plugins.BasePlugin: Returns the plugin object if found. None, otherwise
99+
"""
100+
101+
for p in self._plugins:
102+
if p.name == plugin_name:
103+
return p
104+
105+
return None
106+
107+
def act(self, event, *args, **kwargs):
108+
"""
109+
Act on the specific life cycle event. The action here is to invoke the hook function on all registered plugins.
110+
*args and **kwargs will be passed directly to the plugin's hook functions
111+
112+
:param samtranslator.plugins.LifeCycleEvents event: Event to act upon
113+
:return: Nothing
114+
:raises ValueError: If event is not a valid life cycle event
115+
:raises NameError: If a plugin does not have the hook method defined
116+
:raises Exception: Any exception that a plugin raises
117+
"""
118+
119+
if not isinstance(event, LifeCycleEvents):
120+
raise ValueError("'event' must be an instance of LifeCycleEvents class")
121+
122+
method_name = "on_" + event.name
123+
124+
for plugin in self._plugins:
125+
126+
if not hasattr(plugin, method_name):
127+
raise NameError(
128+
"'{}' method is not found in the plugin with name '{}'".format(method_name, plugin.name)
129+
)
130+
131+
try:
132+
getattr(plugin, method_name)(*args, **kwargs)
133+
except (InvalidResourceException, InvalidDocumentException) as ex:
134+
# Don't need to log these because they don't result in crashes
135+
raise ex
136+
except Exception as ex:
137+
LOG.exception("Plugin '%s' raised an exception: %s", plugin.name, ex)
138+
raise ex
139+
140+
def __len__(self):
141+
"""
142+
Returns the number of plugins registered with this class
143+
144+
:return integer: Number of plugins registered
145+
"""
146+
return len(self._plugins)

samtranslator/translator/translator.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
from samtranslator.plugins.api.default_definition_body_plugin import DefaultDefinitionBodyPlugin
2424
from samtranslator.plugins.application.serverless_app_plugin import ServerlessAppPlugin
2525
from samtranslator.plugins import LifeCycleEvents
26-
from samtranslator.plugins import SamPlugins
26+
from samtranslator.plugins.sam_plugins import SamPlugins
2727
from samtranslator.plugins.globals.globals_plugin import GlobalsPlugin
2828
from samtranslator.plugins.policies.policy_templates_plugin import PolicyTemplatesForResourcePlugin
2929
from samtranslator.policy_template_processor.processor import PolicyTemplatesProcessor

0 commit comments

Comments
 (0)