diff --git a/DESIGN.md b/DESIGN.md index 3ddc07af0..bceb4ccd2 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,3 +1,11 @@ +# Overview + +SAM is called by the CloudFormation Service. CloudFormation recognises the `Transform: AWS::Serverless-2016-10-31` header and invokes the SAM translator. This will then take your SAM template and expand it +into a full fledged CloudFormation Template. The CloudFormation Template that is produced from SAM is the template that is executed by CloudFormation to create/update/delete AWS resources. + +The entry point for SAM starts in the Translator class [here](https://github.com/awslabs/serverless-application-model/blob/develop/samtranslator/translator/translator.py#L29), where SAM iterates through the +template and acts on `AWS::Serverless::*` Type Resources. + # Design decisions Document design decisions here. diff --git a/DEVELOPMENT_GUIDE.rst b/DEVELOPMENT_GUIDE.rst index e929a5dec..034acc25b 100755 --- a/DEVELOPMENT_GUIDE.rst +++ b/DEVELOPMENT_GUIDE.rst @@ -71,4 +71,14 @@ Tests are also a documentation of the success and failure cases, which is crucia .. _excellent cheatsheet: http://python-future.org/compatible_idioms.html .. _pyenv: https://github.com/pyenv/pyenv .. _tox: http://tox.readthedocs.io/en/latest/ -.. _installation instructions: https://github.com/pyenv/pyenv#installation \ No newline at end of file +.. _installation instructions: https://github.com/pyenv/pyenv#installation + +Profiling +--------- + +Install snakeviz `pip install snakeviz` + +``` +python -m cProfile -o sam_profile_results bin/sam-translate.py translate --input-file=tests/translator/input/alexa_skill.yaml --output-file=cfn-template.json +snakeviz sam_profile_results +``` \ No newline at end of file diff --git a/bin/sam-translate.py b/bin/sam-translate.py new file mode 100755 index 000000000..b5ca1d944 --- /dev/null +++ b/bin/sam-translate.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python2 + +"""Convert SAM templates to CloudFormation templates. + +Known limitations: cannot transform CodeUri pointing at local directory. + +Usage: + sam-translate.py --input-file=sam-template.yaml [--output-file=] + +Options: + --input-file= Location of SAM template to transform. + --output-file= Location to store resulting CloudFormation template [default: cfn-template.json]. + +""" +import json +import os + +import boto3 +from docopt import docopt + +from samtranslator.public.translator import ManagedPolicyLoader +from samtranslator.translator.transform import transform +from samtranslator.yaml_helper import yaml_parse + +cli_options = docopt(__doc__) +iam_client = boto3.client('iam') +cwd = os.getcwd() + + +def get_input_output_file_paths(): + input_file_option = cli_options.get('--input-file') + output_file_option = cli_options.get('--output-file') + input_file_path = os.path.join(cwd, input_file_option) + output_file_path = os.path.join(cwd, output_file_option) + + return input_file_path, output_file_path + + +def main(): + input_file_path, output_file_path = get_input_output_file_paths() + + with open(input_file_path, 'r') as f: + sam_template = yaml_parse(f) + + cloud_formation_template = transform( + sam_template, {}, ManagedPolicyLoader(iam_client)) + cloud_formation_template_prettified = json.dumps( + cloud_formation_template, indent=2) + + with open(output_file_path, 'w') as f: + f.write(cloud_formation_template_prettified) + + print('Wrote transformed CloudFormation template to: ' + output_file_path) + + +if __name__ == '__main__': + main() diff --git a/docs/cloudformation_compatibility.rst b/docs/cloudformation_compatibility.rst index 187feb531..968819e1b 100644 --- a/docs/cloudformation_compatibility.rst +++ b/docs/cloudformation_compatibility.rst @@ -91,6 +91,7 @@ Kinesis Property Name Intrinsic(s) Supported Reasons ======================== ================================== ======================== Stream All +Queue All StartingPosition All BatchSize All ======================== ================================== ======================== diff --git a/docs/internals/generated_resources.rst b/docs/internals/generated_resources.rst index 7b3acd3bd..b8323871e 100644 --- a/docs/internals/generated_resources.rst +++ b/docs/internals/generated_resources.rst @@ -220,6 +220,33 @@ AWS::Lambda::Permissions MyFunction\ **MyTrigger**\ Permission AWS::Lambda::EventSourceMapping MyFunction\ **MyTrigger** ================================== ================================ +SQS +^^^^^^^ + +Example: + +.. code:: yaml + + MyFunction: + Type: AWS::Serverless::Function + Properties: + ... + Events: + MyTrigger: + Type: SQS + Properties: + Queue: arn:aws:sqs:us-east-1:123456789012:my-queue + ... + +Additional generated resources: + +================================== ================================ +CloudFormation Resource Type Logical ID +================================== ================================ +AWS::Lambda::Permissions MyFunction\ **MyTrigger**\ Permission +AWS::Lambda::EventSourceMapping MyFunction\ **MyTrigger** +================================== ================================ + DynamoDb ^^^^^^^^ diff --git a/examples/2016-10-31/api_backend/src/index.js b/examples/2016-10-31/api_backend/src/index.js index b910ffc45..e53321cf1 100644 --- a/examples/2016-10-31/api_backend/src/index.js +++ b/examples/2016-10-31/api_backend/src/index.js @@ -6,13 +6,7 @@ const dynamo = new AWS.DynamoDB.DocumentClient(); const tableName = process.env.TABLE_NAME; -const createResponse = (statusCode, body) => { - - return { - statusCode: statusCode, - body: body - } -}; +const createResponse = (statusCode, body) => ({ statusCode, body }); exports.get = (event, context, callback) => { diff --git a/examples/2016-10-31/sqs/.gitignore b/examples/2016-10-31/sqs/.gitignore new file mode 100644 index 000000000..97902fce3 --- /dev/null +++ b/examples/2016-10-31/sqs/.gitignore @@ -0,0 +1 @@ +transformed-cfn-template.yaml \ No newline at end of file diff --git a/examples/2016-10-31/sqs/README.md b/examples/2016-10-31/sqs/README.md new file mode 100644 index 000000000..7c29b96fb --- /dev/null +++ b/examples/2016-10-31/sqs/README.md @@ -0,0 +1,19 @@ +# SQS Event Source Example + +Example SAM template for processing messages on an SQS queue. + +## Running the example + +```bash +# Set YOUR_S3_ARTIFACTS_BUCKET to a bucket you own +YOUR_S3_ARTIFACTS_BUCKET='YOUR_S3_ARTIFACTS_BUCKET'; \ +aws cloudformation package --template-file template.yaml --output-template-file cfn-transformed-template.yaml --s3-bucket $YOUR_S3_ARTIFACTS_BUCKET +aws cloudformation deploy --template-file ./cfn-transformed-template.yaml --stack-name lambda-sqs-processor --capabilities CAPABILITY_IAM +``` + +After your CloudFormation Stack has completed creation, send a message to the SQS queue to see it in action: + +```bash +YOUR_SQS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/123456789012/my-queue; \ +aws sqs send-message --queue-url $YOUR_SQS_QUEUE_URL --message-body '{ "myMessage": "Hello SAM!" }' +``` \ No newline at end of file diff --git a/examples/2016-10-31/sqs/index.js b/examples/2016-10-31/sqs/index.js new file mode 100644 index 000000000..5859fb25c --- /dev/null +++ b/examples/2016-10-31/sqs/index.js @@ -0,0 +1,10 @@ +async function handler (event, context) { + // TODO: Handle message... + const records = event.Records + + console.log(records) + + return {} +} + +module.exports.handler = handler \ No newline at end of file diff --git a/examples/2016-10-31/sqs/template.yaml b/examples/2016-10-31/sqs/template.yaml new file mode 100644 index 000000000..535f7fdac --- /dev/null +++ b/examples/2016-10-31/sqs/template.yaml @@ -0,0 +1,19 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: Example of processing messages on an SQS queue with Lambda +Resources: + MySQSQueueFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: ./index.js + Handler: index.handler + Runtime: nodejs8.10 + Events: + MySQSEvent: + Type: SQS + Properties: + Queue: !GetAtt MySqsQueue.Arn + + MySqsQueue: + Type: AWS::SQS::Queue + Properties: \ No newline at end of file diff --git a/examples/apps/microservice-http-endpoint-python3/index.js b/examples/apps/microservice-http-endpoint-python3/index.js deleted file mode 100644 index 367902b8d..000000000 --- a/examples/apps/microservice-http-endpoint-python3/index.js +++ /dev/null @@ -1,167 +0,0 @@ -/* - This blueprint helps in transitioning Node.js v0.10 functions. It can be run in three modes: - - List: to list all existing Node.js v0.10 functions and their versions in the current region - - Backup: to publish a version of your current deprecated functions $LATEST version. - - Upgrade: to upgrade the runtime field of existing Node.js v0.10 functions $LATEST version from ‘nodejs’ to - ‘node.js4.3’ or ‘node.js6.10’ - - Notes: - - IMPORTANT: This blueprint only upgrades the runtime value of your nodejs v0.10 functions, you should test your - functions to make sure they behave as expected when operating in the new runtime environment. - - Creating Node.js v0.10 functions has been turned off since January 2017. When run in Backup mode, this blueprint - will publish a version of the existing code and configuration of your function. When run in Upgrade mode this - blueprint will upgrade the value of the runtime of $LATEST version of your function. You can point clients of your - existing function to the backup version if required while you work on validating the upgrade. - - If a function fails to backup or upgrade, this Lambda execution will stop and logs will be available in Cloud - Watch for debugging. - - If you have a large number of functions in your account, this function may take multiple invokes to upgrade your - functions. - - If you receive this error message: "The role defined for the function cannot be assumed by Lambda.", please - confirm that the function being upgraded has a correct execution role value and that role exists in IAM. Try - running this blueprint again after the error has been corrected. - - This blueprint is able to upgrade runtimes for the $LATEST version only. Please follow instructions in the - documentation to transition other versions. - - Usage: - 1. Create a function using this blueprint. The functions role should have listFunctions and - updateFunctionConfiguration privileges in its execution role. - 2. To list all existing Node.js v0.10 functions and their versions in the current region, run the function without any - change from the console. This displays the functions and versions as a json list both in the output pane on the console - as well as Cloudwatch logs. - 3. To publish backup versions of your listed functions before upgrading them, make the following changes to the - function's configuration: - a. Change the MODE environment variable's value to Backup. - b. Running this blueprint multiple times in backup mode will add multiple backups to your functions. - 4. To upgrade the runtime field of the listed functions to a newer value, make the following changes to the function’s - configuration: - a. Change the MODE environment variable’s value to Upgrade. - b. Change the TARGET_RUNTIME environment variable’s value to the runtime that you’d like to transition to. Valid - values are nodejs4.3 || nodejs6.10. - c. Change the EXCEPTIONS environment variable’s value to a list of function names to exclude them from being - upgraded. The value should be a comma separated list of function names alone, not ARNs. - d. Run the function from the console. - 5. Repeat step 4 multiple times if you have a lot of functions that need to be upgraded. - 6. Repeat steps 1-5 for all regions you have Lambda functions in. - */ - -'use strict'; - -const AWS = require('aws-sdk'); -const throat = require('throat'); - -const lambda = new AWS.Lambda(); -exports.handler = (event, context, callback) => { - const memory = { Functions: [], Versions: [] }; - const deprecatedRuntime = 'nodejs'; - const targetRuntime = process.env.TARGET_RUNTIME || 'nodejs4.3'; - const mode = (process.env.MODE || 'list').toLowerCase(); - const list = mode === 'list'; - const upgrade = mode === 'upgrade'; - const backup = mode === 'backup'; - const exceptions = process.env.EXCEPTIONS ? process.env.EXCEPTIONS.split(',') : []; - console.log(`Blueprint Deprecated Runtime set to ${deprecatedRuntime}`); - console.log(`Blueprint TARGET_RUNTIME set to ${targetRuntime}`); - console.log(`Blueprint MODE set to ${process.env.MODE}`); - console.log(`Blueprint EXCEPTIONS set to ${process.env.EXCEPTIONS}`); - - function report() { - const formatExample = { DeprecatedFunctionName: ['DeprecatedVersion1', 'DeprecatedVersion2'] }; - const functionNames = memory.Functions.map((fn) => { - const obj = {}; - obj[`${fn.FunctionName}`] = JSON.stringify(memory.Versions.filter(vs => vs.FunctionName === `${fn.FunctionName}`).map(vs => vs.Version)); - return obj; - }); - if (functionNames.length) { - functionNames.unshift(formatExample); - console.log('Printing deprecated functions and their corresponding deprecated versions.' + - 'The following functions runtimes will be upgraded. Example format: ', functionNames); - } else { - console.log('No deprecated functions found.'); - } - } - - function backupFunctions(functions) { - return Promise.all(functions.map(throat(1, (fn) => { - console.log(`Starting backup of function ${fn}`); - const params = { - FunctionName: fn, - Description: 'Node 0.10 Deprecation Blueprint Backup', - }; - return lambda.publishVersion(params).promise(); - }))); - } - - function upgradeFunctions(functions) { - return Promise.all(functions.map(throat(1, (fn) => { - console.log(`Starting runtime upgrade of function ${fn}`); - const params = { - FunctionName: fn, - Runtime: targetRuntime, - }; - return lambda.updateFunctionConfiguration(params).promise(); - }))); - } - - function getVersions(functions) { - return Promise.all(functions.map(throat(1, (fn) => { - const params = { - FunctionName: fn, - }; - return lambda.listVersionsByFunction(params).promise(); - }))); - } - - function getFunctions(params) { - lambda.listFunctions(params, (err, data) => { - if (err) { - callback(err, err.stack); - } else { - Array.prototype.push.apply(memory.Functions, data.Functions.filter((item) => item.Runtime === deprecatedRuntime && - exceptions.indexOf(item.FunctionName) === -1)); - if (data.NextMarker) { - const nextListFunctionsParams = { - Marker: data.NextMarker, - MaxItems: 50, - }; - setTimeout(() => getFunctions(nextListFunctionsParams), 100); - } else { - // retrieved all functions - console.log(`Total deprecated functions retreived: ${memory.Functions.length}`); - getVersions(memory.Functions.map(fn => fn.FunctionName)).then((versions) => { - memory.Versions = versions; - if (list) { - report(); - console.log('Report Complete.'); - } else if (backup) { - console.log(`Starting Function Backup Operation for ${memory.Functions.length} deprecated functions.`); - backupFunctions(memory.Functions.map(fn => fn.FunctionName)) - .then(() => { - console.log('Function Backup Operation Complete. See CloudWatch logs for Errors' + - ' that may have occurred.'); - }) - .catch((error) => { - console.log(error); - }); - } else if (upgrade) { - console.log(`Starting Function Upgrade Operation for ${memory.Functions.length} deprecated functions.`); - upgradeFunctions(memory.Functions.map(fn => fn.FunctionName)) - .then(() => { - console.log('Function Upgrade Operation Complete. See CloudWatch logs for Errors' + - ' that may have occurred.'); - }) - .catch((error) => { - console.log(error); - }); - } else { - console.log('no MODE environment variable specified.'); - } - }); - } - } - }); - } - const starterParams = { - MaxItems: 50, - }; - getFunctions(starterParams); -}; diff --git a/examples/apps/microservice-http-endpoint-python3/lambda_function.py b/examples/apps/microservice-http-endpoint-python3/lambda_function.py index c997e58dc..838fd389d 100644 --- a/examples/apps/microservice-http-endpoint-python3/lambda_function.py +++ b/examples/apps/microservice-http-endpoint-python3/lambda_function.py @@ -1,9 +1,11 @@ import boto3 import json +import os print('Loading function') dynamo = boto3.client('dynamodb') +table_name = os.environ['TABLE_NAME'] def respond(err, res=None): @@ -21,18 +23,19 @@ def lambda_handler(event, context): access to the request and response payload, including headers and status code. - To scan a DynamoDB table, make a GET request with the TableName as a - query string parameter. To put, update, or delete an item, make a POST, - PUT, or DELETE request respectively, passing in the payload to the - DynamoDB API as a JSON body. + TableName provided by template.yaml. + + To scan a DynamoDB table, make a GET request with optional query string parameter. + To put, update, or delete an item, make a POST, PUT, or DELETE request respectively, + passing in the payload to the DynamoDB API as a JSON body. ''' - #print("Received event: " + json.dumps(event, indent=2)) + print("Received event: " + json.dumps(event, indent=2)) operations = { - 'DELETE': lambda dynamo, x: dynamo.delete_item(**x), - 'GET': lambda dynamo, x: dynamo.scan(**x), - 'POST': lambda dynamo, x: dynamo.put_item(**x), - 'PUT': lambda dynamo, x: dynamo.update_item(**x), + 'DELETE': lambda dynamo, x: dynamo.delete_item(TableName=table_name, **x), + 'GET': lambda dynamo, x: dynamo.scan(TableName=table_name, **x) if x else dynamo.scan(TableName=table_name), + 'POST': lambda dynamo, x: dynamo.put_item(TableName=table_name, **x), + 'PUT': lambda dynamo, x: dynamo.update_item(TableName=table_name, **x), } operation = event['httpMethod'] diff --git a/examples/apps/microservice-http-endpoint-python3/template.yaml b/examples/apps/microservice-http-endpoint-python3/template.yaml index 9ae466e77..28045b93d 100644 --- a/examples/apps/microservice-http-endpoint-python3/template.yaml +++ b/examples/apps/microservice-http-endpoint-python3/template.yaml @@ -3,17 +3,30 @@ Transform: 'AWS::Serverless-2016-10-31' Description: >- A simple backend (read/write to DynamoDB) with a RESTful API endpoint using Amazon API Gateway. +Globals: + #https://github.com/awslabs/serverless-application-model/blob/develop/docs/globals.rst + Function: + Runtime: python3.6 + MemorySize: 512 + #VpcConfig: + #Tracing: + #AutoPublishAlias: + Environment: + Variables: + TABLE_NAME: + Ref: Table + + + Resources: microservicehttpendpointpython3: Type: 'AWS::Serverless::Function' Properties: Handler: lambda_function.lambda_handler - Runtime: python3.6 CodeUri: . Description: >- A simple backend (read/write to DynamoDB) with a RESTful API endpoint using Amazon API Gateway. - MemorySize: 512 Timeout: 10 Policies: - Version: '2012-10-17' @@ -32,10 +45,15 @@ Resources: - Ref: 'AWS::Region' - ':' - Ref: 'AWS::AccountId' - - ':table/*' + - ':table/' + - Ref: Table Events: Api1: Type: Api Properties: Path: /MyResource Method: ANY + + Table: + Type: AWS::Serverless::SimpleTable + diff --git a/examples/apps/microservice-http-endpoint-python3/test-output.log b/examples/apps/microservice-http-endpoint-python3/test-output.log new file mode 100644 index 000000000..c15555349 --- /dev/null +++ b/examples/apps/microservice-http-endpoint-python3/test-output.log @@ -0,0 +1,70 @@ +❯ ./test.sh https://vxldx8ck9i.execute-api.us-west-2.amazonaws.com/Prod/MyResource ./test-payload.json ++ http POST https://vxldx8ck9i.execute-api.us-west-2.amazonaws.com/Prod/MyResource Item:=@./test-payload.json +HTTP/1.1 200 OK +Connection: keep-alive +Content-Length: 414 +Content-Type: application/json +Date: Thu, 03 May 2018 21:13:56 GMT +Via: 1.1 d159f3d447f69796b1a04dffe8d243c0.cloudfront.net (CloudFront) +X-Amz-Cf-Id: 60kAxGheadAiJo-ksjknQr-rAh-WZ2EaJXMT_cYH5nzgwIudXHUu9g== +X-Amzn-Trace-Id: Root=1-5aeb7b94-457eb371f443bfc06c5dadb3 +X-Cache: Miss from cloudfront +x-amz-apigw-id: GVA_OEZIPHcFleg= +x-amzn-RequestId: e3e943ab-4f16-11e8-af74-e7a92312c1b8 + +{ + "ResponseMetadata": { + "HTTPHeaders": { + "connection": "keep-alive", + "content-length": "2", + "content-type": "application/x-amz-json-1.0", + "date": "Thu, 03 May 2018 21:13:56 GMT", + "server": "Server", + "x-amz-crc32": "2745614147", + "x-amzn-requestid": "VQGS8JQIH2G9CDO8P65U6HMKEFVV4KQNSO5AEMVJF66Q9ASUAAJG" + }, + "HTTPStatusCode": 200, + "RequestId": "VQGS8JQIH2G9CDO8P65U6HMKEFVV4KQNSO5AEMVJF66Q9ASUAAJG", + "RetryAttempts": 0 + } +} + ++ http GET https://vxldx8ck9i.execute-api.us-west-2.amazonaws.com/Prod/MyResource +HTTP/1.1 200 OK +Connection: keep-alive +Content-Length: 485 +Content-Type: application/json +Date: Thu, 03 May 2018 21:13:57 GMT +Via: 1.1 0419e0c9f6ea9a42ab82db8feb536aee.cloudfront.net (CloudFront) +X-Amz-Cf-Id: Nsflnq2N-dxYOskE8ui1ULydLO9fg7-HaCmbtjMwZ1A7Pr_wgEJUhA== +X-Amzn-Trace-Id: Root=1-5aeb7b95-f0ceaed903bf48b8605b6899 +X-Cache: Miss from cloudfront +x-amz-apigw-id: GVA_UHO5PHcFo7w= +x-amzn-RequestId: e444d07b-4f16-11e8-8db6-6940d66a847c + +{ + "Count": 1, + "Items": [ + { + "id": { + "S": "id_string" + } + } + ], + "ResponseMetadata": { + "HTTPHeaders": { + "connection": "keep-alive", + "content-length": "63", + "content-type": "application/x-amz-json-1.0", + "date": "Thu, 03 May 2018 21:13:57 GMT", + "server": "Server", + "x-amz-crc32": "3041197985", + "x-amzn-requestid": "9NFOFDHCCR5OE0V81IUC70Q6FNVV4KQNSO5AEMVJF66Q9ASUAAJG" + }, + "HTTPStatusCode": 200, + "RequestId": "9NFOFDHCCR5OE0V81IUC70Q6FNVV4KQNSO5AEMVJF66Q9ASUAAJG", + "RetryAttempts": 0 + }, + "ScannedCount": 1 +} + diff --git a/examples/apps/microservice-http-endpoint-python3/test-payload.json b/examples/apps/microservice-http-endpoint-python3/test-payload.json new file mode 100644 index 000000000..01908265f --- /dev/null +++ b/examples/apps/microservice-http-endpoint-python3/test-payload.json @@ -0,0 +1,5 @@ +{ + "id": { + "S": "id_string" + } +} diff --git a/examples/apps/microservice-http-endpoint-python3/test.sh b/examples/apps/microservice-http-endpoint-python3/test.sh new file mode 100755 index 000000000..96caa92b6 --- /dev/null +++ b/examples/apps/microservice-http-endpoint-python3/test.sh @@ -0,0 +1,19 @@ +#/bin/bash -e + +# Test invocation using httpie (http) in place of curl for syntax simplicity +# +# Args: $1: API GW path +# $2 json payload data file +# +# eg: ./test.sh https://.execute-api..amazonaws.com/Prod/MyResource ./test-payload.json + +set -x + +# Write one item + +http POST $1 Item:=@$2 + +# Read all back + +http GET $1 + diff --git a/requirements/dev.txt b/requirements/dev.txt index 47c8104d5..aba58807c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -12,3 +12,6 @@ mock>=2.0.0 nose>=1.3.7 parameterized>=0.6.1 requests>=2.11.1 + +# CLI requirements +docopt>=0.6.2 \ No newline at end of file diff --git a/samtranslator/model/eventsources/pull.py b/samtranslator/model/eventsources/pull.py index 7a692d585..e2e309237 100644 --- a/samtranslator/model/eventsources/pull.py +++ b/samtranslator/model/eventsources/pull.py @@ -3,21 +3,23 @@ from samtranslator.model.lambda_ import LambdaEventSourceMapping from samtranslator.translator.arn_generator import ArnGenerator +from samtranslator.model.exceptions import InvalidEventException class PullEventSource(ResourceMacro): """Base class for pull event sources for SAM Functions. - The pull events are the streams--Kinesis and DynamoDB Streams. Both of these correspond to an EventSourceMapping in - Lambda, and require that the execution role be given to Kinesis or DynamoDB Streams, respectively. + The pull events are Kinesis Streams, DynamoDB Streams, and SQS Queues. All of these correspond to an EventSourceMapping in + Lambda, and require that the execution role be given to Kinesis Streams, DynamoDB Streams, or SQS Queues, respectively. :cvar str policy_arn: The ARN of the AWS managed role policy corresponding to this pull event source """ resource_type = None property_types = { - 'Stream': PropertyType(True, is_str()), - 'BatchSize': PropertyType(False, is_type(int)), - 'StartingPosition': PropertyType(True, is_str()) + 'Stream': PropertyType(False, is_str()), + 'Queue': PropertyType(False, is_str()), + 'BatchSize': PropertyType(False, is_type(int)), + 'StartingPosition': PropertyType(False, is_str()) } def get_policy_arn(self): @@ -32,7 +34,7 @@ def to_cloudformation(self, **kwargs): :rtype: list """ function = kwargs.get('function') - + if not function: raise TypeError("Missing required keyword argument: function") @@ -40,15 +42,23 @@ def to_cloudformation(self, **kwargs): lambda_eventsourcemapping = LambdaEventSourceMapping(self.logical_id) resources.append(lambda_eventsourcemapping) - + try: # Name will not be available for Alias resources function_name_or_arn = function.get_runtime_attr("name") except NotImplementedError: function_name_or_arn = function.get_runtime_attr("arn") + if not self.Stream and not self.Queue: + raise InvalidEventException( + self.relative_id, "No Queue (for SQS) or Stream (for Kinesis or DynamoDB) provided.") + + if self.Stream and not self.StartingPosition: + raise InvalidEventException( + self.relative_id, "StartingPosition is required for Kinesis and DynamoDB.") + lambda_eventsourcemapping.FunctionName = function_name_or_arn - lambda_eventsourcemapping.EventSourceArn = self.Stream + lambda_eventsourcemapping.EventSourceArn = self.Stream or self.Queue lambda_eventsourcemapping.StartingPosition = self.StartingPosition lambda_eventsourcemapping.BatchSize = self.BatchSize @@ -82,3 +92,11 @@ class DynamoDB(PullEventSource): def get_policy_arn(self): return ArnGenerator.generate_aws_managed_policy_arn('service-role/AWSLambdaDynamoDBExecutionRole') + + +class SQS(PullEventSource): + """SQS Queue event source.""" + resource_type = 'SQS' + + def get_policy_arn(self): + return ArnGenerator.generate_aws_managed_policy_arn('service-role/AWSLambdaSQSExecutionRole') diff --git a/samtranslator/model/eventsources/push.py b/samtranslator/model/eventsources/push.py index c155b5a19..f04198ef9 100644 --- a/samtranslator/model/eventsources/push.py +++ b/samtranslator/model/eventsources/push.py @@ -429,11 +429,12 @@ def _get_permission(self, resources_to_link, stage, suffix): raise RuntimeError("Could not add permission to lambda function.") path = self.Path.replace('{proxy+}', '*') + method = '*' if self.Method.lower() == 'any' else self.Method.upper() api_id = self.RestApiId # RestApiId can be a simple string or intrinsic function like !Ref. Using Fn::Sub will handle both cases - resource = '${__ApiId__}/' + '${__Stage__}/' + self.Method.upper() + path + resource = '${__ApiId__}/' + '${__Stage__}/' + method + path partition = ArnGenerator.get_partition_name() source_arn = fnSub(ArnGenerator.generate_arn(partition=partition, service='execute-api', resource=resource), {"__ApiId__": api_id, "__Stage__": stage}) diff --git a/samtranslator/model/lambda_.py b/samtranslator/model/lambda_.py index f1900a9d7..bf0fe6b12 100644 --- a/samtranslator/model/lambda_.py +++ b/samtranslator/model/lambda_.py @@ -61,7 +61,7 @@ class LambdaEventSourceMapping(Resource): 'Enabled': PropertyType(False, is_type(bool)), 'EventSourceArn': PropertyType(True, is_str()), 'FunctionName': PropertyType(True, is_str()), - 'StartingPosition': PropertyType(True, is_str()) + 'StartingPosition': PropertyType(False, is_str()) } runtime_attrs = { diff --git a/samtranslator/parser/parser.py b/samtranslator/parser/parser.py index d1becc621..41cc1313a 100644 --- a/samtranslator/parser/parser.py +++ b/samtranslator/parser/parser.py @@ -2,6 +2,7 @@ from samtranslator.validator.validator import SamTemplateValidator from samtranslator.model import ResourceTypeResolver, sam_resources from samtranslator.plugins import LifeCycleEvents +import logging class Parser: def __init__(self): @@ -25,4 +26,11 @@ def _validate(self, sam_template, parameter_values): raise InvalidDocumentException( [InvalidTemplateException("'Resources' section is required")]) - SamTemplateValidator.validate(sam_template) + validation_errors = SamTemplateValidator.validate(sam_template) + has_errors = len(validation_errors) + + if has_errors: + # NOTE: eventually we will throw on invalid schema + # raise InvalidDocumentException([InvalidTemplateException(validation_errors)]) + logging.warning( + "JSON_VALIDATION_WARNING: {0}".format(validation_errors)) diff --git a/samtranslator/validator/sam_schema/schema.json b/samtranslator/validator/sam_schema/schema.json index cdcb08d4b..cc2ed7e04 100644 --- a/samtranslator/validator/sam_schema/schema.json +++ b/samtranslator/validator/sam_schema/schema.json @@ -381,6 +381,9 @@ { "$ref": "#/definitions/AWS::Serverless::Function.KinesisEvent" }, + { + "$ref": "#/definitions/AWS::Serverless::Function.SQSEvent" + }, { "$ref": "#/definitions/AWS::Serverless::Function.DynamoDBEvent" }, @@ -481,6 +484,27 @@ ], "type": "object" }, + "AWS::Serverless::Function.SQSEvent": { + "additionalProperties": false, + "properties": { + "BatchSize": { + "type": "number" + }, + "Queue": { + "anyOf": [{ + "type": "string" + }, + { + "type": "object" + } + ] + } + }, + "required": [ + "Queue" + ], + "type": "object" + }, "AWS::Serverless::Function.S3Event": { "additionalProperties": false, "properties": { diff --git a/tests/translator/yaml_helper.py b/samtranslator/yaml_helper.py similarity index 100% rename from tests/translator/yaml_helper.py rename to samtranslator/yaml_helper.py diff --git a/tests/translator/input/error_missing_queue.yaml b/tests/translator/input/error_missing_queue.yaml new file mode 100644 index 000000000..499453c4a --- /dev/null +++ b/tests/translator/input/error_missing_queue.yaml @@ -0,0 +1,17 @@ +# File: sam.yml +# Version: 0.9 + +AWSTemplateFormatVersion: '2010-09-09' +Parameters: {} +Resources: + SQSFunction: + Type: 'AWS::Serverless::Function' + Properties: + CodeUri: s3://sam-demo-bucket/queues.zip + Handler: queue.sqs_handler + Runtime: python2.7 + Events: + MySqsQueue: + Type: SQS + Properties: + BatchSize: 10 diff --git a/tests/translator/input/error_missing_startingposition.yaml b/tests/translator/input/error_missing_startingposition.yaml new file mode 100644 index 000000000..fee16ded0 --- /dev/null +++ b/tests/translator/input/error_missing_startingposition.yaml @@ -0,0 +1,13 @@ +Resources: + KinesisFunction: + Type: 'AWS::Serverless::Function' + Properties: + CodeUri: s3://sam-demo-bucket/streams.zip + Handler: stream.kinesis_handler + Runtime: python2.7 + Events: + MyKinesisStream: + Type: Kinesis + Properties: + Stream: arn:aws:kinesis:us-west-2:012345678901:stream/my-stream + BatchSize: 100 \ No newline at end of file diff --git a/tests/translator/input/error_missing_stream.yaml b/tests/translator/input/error_missing_stream.yaml new file mode 100644 index 000000000..56a22aedb --- /dev/null +++ b/tests/translator/input/error_missing_stream.yaml @@ -0,0 +1,15 @@ +AWSTemplateFormatVersion: '2010-09-09' +Parameters: {} +Resources: + DynamoDBFunction: + Type: 'AWS::Serverless::Function' + Properties: + CodeUri: s3://sam-demo-bucket/streams.zip + Handler: stream.ddb_handler + Runtime: python2.7 + Events: + MyDDBStream: + Type: DynamoDB + Properties: + BatchSize: 200 + StartingPosition: LATEST \ No newline at end of file diff --git a/tests/translator/input/sqs.yaml b/tests/translator/input/sqs.yaml new file mode 100644 index 000000000..8ee02599a --- /dev/null +++ b/tests/translator/input/sqs.yaml @@ -0,0 +1,13 @@ +Resources: + SQSFunction: + Type: 'AWS::Serverless::Function' + Properties: + CodeUri: s3://sam-demo-bucket/queues.zip + Handler: queue.sqs_handler + Runtime: python2.7 + Events: + MySqsQueue: + Type: SQS + Properties: + Queue: arn:aws:sqs:us-west-2:012345678901:my-queue + BatchSize: 10 diff --git a/tests/translator/output/api_with_cors.json b/tests/translator/output/api_with_cors.json index b0adb9344..0791fc4f4 100644 --- a/tests/translator/output/api_with_cors.json +++ b/tests/translator/output/api_with_cors.json @@ -78,7 +78,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/foo", + "arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/foo", { "__Stage__": "Prod", "__ApiId__": { @@ -428,7 +428,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/foo", + "arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/foo", { "__Stage__": "*", "__ApiId__": { diff --git a/tests/translator/output/aws-cn/api_with_cors.json b/tests/translator/output/aws-cn/api_with_cors.json index 5637516ec..e3d3f3a3f 100644 --- a/tests/translator/output/aws-cn/api_with_cors.json +++ b/tests/translator/output/aws-cn/api_with_cors.json @@ -78,7 +78,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws-cn:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/foo", + "arn:aws-cn:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/foo", { "__Stage__": "Prod", "__ApiId__": { @@ -444,7 +444,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws-cn:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/foo", + "arn:aws-cn:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/foo", { "__Stage__": "*", "__ApiId__": { diff --git a/tests/translator/output/aws-cn/error_missing_queue.json b/tests/translator/output/aws-cn/error_missing_queue.json new file mode 100644 index 000000000..f92bbf80e --- /dev/null +++ b/tests/translator/output/aws-cn/error_missing_queue.json @@ -0,0 +1,6 @@ +{ + "errors": [{ + "errorMessage": "Resource with id [SQSFunction] is invalid. Event with id [MySqsQueue] is invalid. No Queue (for SQS) or Stream (for Kinesis or DynamoDB) provided." + }], + "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [SQSFunction] is invalid. Event with id [MySqsQueue] is invalid. No Queue (for SQS) or Stream (for Kinesis or DynamoDB) provided." +} \ No newline at end of file diff --git a/tests/translator/output/aws-cn/error_missing_stream.json b/tests/translator/output/aws-cn/error_missing_stream.json new file mode 100644 index 000000000..d8925a773 --- /dev/null +++ b/tests/translator/output/aws-cn/error_missing_stream.json @@ -0,0 +1,6 @@ +{ + "errors": [{ + "errorMessage": "Resource with id [DynamoDBFunction] is invalid. Event with id [MyDDBStream] is invalid. No Queue (for SQS) or Stream (for Kinesis or DynamoDB) provided." + }], + "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [DynamoDBFunction] is invalid. Event with id [MyDDBStream] is invalid. No Queue (for SQS) or Stream (for Kinesis or DynamoDB) provided." +} \ No newline at end of file diff --git a/tests/translator/output/aws-cn/implicit_api.json b/tests/translator/output/aws-cn/implicit_api.json index 4b5539716..b59c4bb0d 100644 --- a/tests/translator/output/aws-cn/implicit_api.json +++ b/tests/translator/output/aws-cn/implicit_api.json @@ -98,7 +98,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws-cn:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/*", + "arn:aws-cn:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/*", { "__Stage__": "Prod", "__ApiId__": { @@ -200,7 +200,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws-cn:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/*", + "arn:aws-cn:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/*", { "__Stage__": "*", "__ApiId__": { diff --git a/tests/translator/output/aws-cn/implicit_api_with_serverless_rest_api_resource.json b/tests/translator/output/aws-cn/implicit_api_with_serverless_rest_api_resource.json index 4b5539716..b59c4bb0d 100644 --- a/tests/translator/output/aws-cn/implicit_api_with_serverless_rest_api_resource.json +++ b/tests/translator/output/aws-cn/implicit_api_with_serverless_rest_api_resource.json @@ -98,7 +98,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws-cn:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/*", + "arn:aws-cn:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/*", { "__Stage__": "Prod", "__ApiId__": { @@ -200,7 +200,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws-cn:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/*", + "arn:aws-cn:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/*", { "__Stage__": "*", "__ApiId__": { diff --git a/tests/translator/output/aws-cn/sqs.json b/tests/translator/output/aws-cn/sqs.json new file mode 100644 index 000000000..29e529af5 --- /dev/null +++ b/tests/translator/output/aws-cn/sqs.json @@ -0,0 +1,58 @@ +{ + "Resources": { + "SQSFunctionRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "ManagedPolicyArns": [ + "arn:aws-cn:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + "arn:aws-cn:iam::aws:policy/service-role/AWSLambdaSQSExecutionRole" + ], + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] + } + }] + } + } + }, + "SQSFunction": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": "sam-demo-bucket", + "S3Key": "queues.zip" + }, + "Handler": "queue.sqs_handler", + "Role": { + "Fn::GetAtt": [ + "SQSFunctionRole", + "Arn" + ] + }, + "Runtime": "python2.7", + "Tags": [{ + "Value": "SAM", + "Key": "lambda:createdBy" + }] + } + }, + "SQSFunctionMySqsQueue": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties": { + "BatchSize": 10, + "EventSourceArn": "arn:aws:sqs:us-west-2:012345678901:my-queue", + "FunctionName": { + "Ref": "SQSFunction" + } + } + } + } +} \ No newline at end of file diff --git a/tests/translator/output/aws-cn/streams.json b/tests/translator/output/aws-cn/streams.json index f760304aa..f4c82223c 100644 --- a/tests/translator/output/aws-cn/streams.json +++ b/tests/translator/output/aws-cn/streams.json @@ -1,120 +1,112 @@ { "Resources": { "DynamoDBFunction": { - "Type": "AWS::Lambda::Function", + "Type": "AWS::Lambda::Function", "Properties": { "Code": { "S3Bucket": "sam-demo-bucket", "S3Key": "streams.zip" - }, - "Handler": "stream.ddb_handler", + }, + "Handler": "stream.ddb_handler", "Role": { "Fn::GetAtt": [ - "DynamoDBFunctionRole", + "DynamoDBFunctionRole", "Arn" ] - }, - "Runtime": "python2.7", - "Tags": [ - { - "Value": "SAM", - "Key": "lambda:createdBy" - } - ] + }, + "Runtime": "python2.7", + "Tags": [{ + "Value": "SAM", + "Key": "lambda:createdBy" + }] } - }, + }, "KinesisFunction": { - "Type": "AWS::Lambda::Function", + "Type": "AWS::Lambda::Function", "Properties": { "Code": { "S3Bucket": "sam-demo-bucket", "S3Key": "streams.zip" - }, - "Handler": "stream.kinesis_handler", + }, + "Handler": "stream.kinesis_handler", "Role": { "Fn::GetAtt": [ - "KinesisFunctionRole", + "KinesisFunctionRole", "Arn" ] - }, - "Runtime": "python2.7", - "Tags": [ - { - "Value": "SAM", - "Key": "lambda:createdBy" - } - ] + }, + "Runtime": "python2.7", + "Tags": [{ + "Value": "SAM", + "Key": "lambda:createdBy" + }] } - }, + }, "DynamoDBFunctionMyDDBStream": { - "Type": "AWS::Lambda::EventSourceMapping", + "Type": "AWS::Lambda::EventSourceMapping", "Properties": { - "BatchSize": 200, + "BatchSize": 200, "EventSourceArn": "arn:aws:dynamodb:us-west-2:012345678901:table/TestTable/stream/2015-05-11T21:21:33.291", "FunctionName": { "Ref": "DynamoDBFunction" - }, + }, "StartingPosition": "LATEST" } - }, + }, "DynamoDBFunctionRole": { - "Type": "AWS::IAM::Role", + "Type": "AWS::IAM::Role", "Properties": { "ManagedPolicyArns": [ "arn:aws-cn:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", "arn:aws-cn:iam::aws:policy/service-role/AWSLambdaDynamoDBExecutionRole" - ], + ], "AssumeRolePolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Action": [ - "sts:AssumeRole" - ], - "Effect": "Allow", - "Principal": { - "Service": [ - "lambda.amazonaws.com" - ] - } + "Version": "2012-10-17", + "Statement": [{ + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] } - ] + }] } } - }, + }, "KinesisFunctionMyKinesisStream": { - "Type": "AWS::Lambda::EventSourceMapping", + "Type": "AWS::Lambda::EventSourceMapping", "Properties": { - "BatchSize": 100, + "BatchSize": 100, "EventSourceArn": "arn:aws:kinesis:us-west-2:012345678901:stream/my-stream", "FunctionName": { "Ref": "KinesisFunction" - }, + }, "StartingPosition": "TRIM_HORIZON" } - }, + }, "KinesisFunctionRole": { - "Type": "AWS::IAM::Role", + "Type": "AWS::IAM::Role", "Properties": { "ManagedPolicyArns": [ "arn:aws-cn:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", "arn:aws-cn:iam::aws:policy/service-role/AWSLambdaKinesisExecutionRole" - ], + ], "AssumeRolePolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Action": [ - "sts:AssumeRole" - ], - "Effect": "Allow", - "Principal": { - "Service": [ - "lambda.amazonaws.com" - ] - } + "Version": "2012-10-17", + "Statement": [{ + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] } - ] + }] } } } diff --git a/tests/translator/output/aws-us-gov/api_with_cors.json b/tests/translator/output/aws-us-gov/api_with_cors.json index cac6ceb3c..3ddac1895 100644 --- a/tests/translator/output/aws-us-gov/api_with_cors.json +++ b/tests/translator/output/aws-us-gov/api_with_cors.json @@ -78,7 +78,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws-us-gov:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/foo", + "arn:aws-us-gov:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/foo", { "__Stage__": "Prod", "__ApiId__": { @@ -444,7 +444,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws-us-gov:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/foo", + "arn:aws-us-gov:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/foo", { "__Stage__": "*", "__ApiId__": { diff --git a/tests/translator/output/aws-us-gov/error_missing_queue.json b/tests/translator/output/aws-us-gov/error_missing_queue.json new file mode 100644 index 000000000..f92bbf80e --- /dev/null +++ b/tests/translator/output/aws-us-gov/error_missing_queue.json @@ -0,0 +1,6 @@ +{ + "errors": [{ + "errorMessage": "Resource with id [SQSFunction] is invalid. Event with id [MySqsQueue] is invalid. No Queue (for SQS) or Stream (for Kinesis or DynamoDB) provided." + }], + "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [SQSFunction] is invalid. Event with id [MySqsQueue] is invalid. No Queue (for SQS) or Stream (for Kinesis or DynamoDB) provided." +} \ No newline at end of file diff --git a/tests/translator/output/aws-us-gov/error_missing_stream.json b/tests/translator/output/aws-us-gov/error_missing_stream.json new file mode 100644 index 000000000..d8925a773 --- /dev/null +++ b/tests/translator/output/aws-us-gov/error_missing_stream.json @@ -0,0 +1,6 @@ +{ + "errors": [{ + "errorMessage": "Resource with id [DynamoDBFunction] is invalid. Event with id [MyDDBStream] is invalid. No Queue (for SQS) or Stream (for Kinesis or DynamoDB) provided." + }], + "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [DynamoDBFunction] is invalid. Event with id [MyDDBStream] is invalid. No Queue (for SQS) or Stream (for Kinesis or DynamoDB) provided." +} \ No newline at end of file diff --git a/tests/translator/output/aws-us-gov/implicit_api.json b/tests/translator/output/aws-us-gov/implicit_api.json index 7093d981b..e8a85e047 100644 --- a/tests/translator/output/aws-us-gov/implicit_api.json +++ b/tests/translator/output/aws-us-gov/implicit_api.json @@ -98,7 +98,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws-us-gov:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/*", + "arn:aws-us-gov:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/*", { "__Stage__": "Prod", "__ApiId__": { @@ -210,7 +210,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws-us-gov:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/*", + "arn:aws-us-gov:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/*", { "__Stage__": "*", "__ApiId__": { diff --git a/tests/translator/output/aws-us-gov/implicit_api_with_serverless_rest_api_resource.json b/tests/translator/output/aws-us-gov/implicit_api_with_serverless_rest_api_resource.json index 7093d981b..e8a85e047 100644 --- a/tests/translator/output/aws-us-gov/implicit_api_with_serverless_rest_api_resource.json +++ b/tests/translator/output/aws-us-gov/implicit_api_with_serverless_rest_api_resource.json @@ -98,7 +98,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws-us-gov:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/*", + "arn:aws-us-gov:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/*", { "__Stage__": "Prod", "__ApiId__": { @@ -210,7 +210,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws-us-gov:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/*", + "arn:aws-us-gov:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/*", { "__Stage__": "*", "__ApiId__": { diff --git a/tests/translator/output/aws-us-gov/sqs.json b/tests/translator/output/aws-us-gov/sqs.json new file mode 100644 index 000000000..f58f3df02 --- /dev/null +++ b/tests/translator/output/aws-us-gov/sqs.json @@ -0,0 +1,58 @@ +{ + "Resources": { + "SQSFunctionRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "ManagedPolicyArns": [ + "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaSQSExecutionRole" + ], + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] + } + }] + } + } + }, + "SQSFunction": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": "sam-demo-bucket", + "S3Key": "queues.zip" + }, + "Handler": "queue.sqs_handler", + "Role": { + "Fn::GetAtt": [ + "SQSFunctionRole", + "Arn" + ] + }, + "Runtime": "python2.7", + "Tags": [{ + "Value": "SAM", + "Key": "lambda:createdBy" + }] + } + }, + "SQSFunctionMySqsQueue": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties": { + "BatchSize": 10, + "EventSourceArn": "arn:aws:sqs:us-west-2:012345678901:my-queue", + "FunctionName": { + "Ref": "SQSFunction" + } + } + } + } +} \ No newline at end of file diff --git a/tests/translator/output/aws-us-gov/streams.json b/tests/translator/output/aws-us-gov/streams.json index 7d6098db6..d4f718822 100644 --- a/tests/translator/output/aws-us-gov/streams.json +++ b/tests/translator/output/aws-us-gov/streams.json @@ -1,120 +1,112 @@ { "Resources": { "DynamoDBFunction": { - "Type": "AWS::Lambda::Function", + "Type": "AWS::Lambda::Function", "Properties": { "Code": { "S3Bucket": "sam-demo-bucket", "S3Key": "streams.zip" - }, - "Handler": "stream.ddb_handler", + }, + "Handler": "stream.ddb_handler", "Role": { "Fn::GetAtt": [ - "DynamoDBFunctionRole", + "DynamoDBFunctionRole", "Arn" ] - }, - "Runtime": "python2.7", - "Tags": [ - { - "Value": "SAM", - "Key": "lambda:createdBy" - } - ] + }, + "Runtime": "python2.7", + "Tags": [{ + "Value": "SAM", + "Key": "lambda:createdBy" + }] } - }, + }, "KinesisFunction": { - "Type": "AWS::Lambda::Function", + "Type": "AWS::Lambda::Function", "Properties": { "Code": { "S3Bucket": "sam-demo-bucket", "S3Key": "streams.zip" - }, - "Handler": "stream.kinesis_handler", + }, + "Handler": "stream.kinesis_handler", "Role": { "Fn::GetAtt": [ - "KinesisFunctionRole", + "KinesisFunctionRole", "Arn" ] - }, - "Runtime": "python2.7", - "Tags": [ - { - "Value": "SAM", - "Key": "lambda:createdBy" - } - ] + }, + "Runtime": "python2.7", + "Tags": [{ + "Value": "SAM", + "Key": "lambda:createdBy" + }] } - }, + }, "DynamoDBFunctionMyDDBStream": { - "Type": "AWS::Lambda::EventSourceMapping", + "Type": "AWS::Lambda::EventSourceMapping", "Properties": { - "BatchSize": 200, + "BatchSize": 200, "EventSourceArn": "arn:aws:dynamodb:us-west-2:012345678901:table/TestTable/stream/2015-05-11T21:21:33.291", "FunctionName": { "Ref": "DynamoDBFunction" - }, + }, "StartingPosition": "LATEST" } - }, + }, "DynamoDBFunctionRole": { - "Type": "AWS::IAM::Role", + "Type": "AWS::IAM::Role", "Properties": { "ManagedPolicyArns": [ "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaDynamoDBExecutionRole" - ], + ], "AssumeRolePolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Action": [ - "sts:AssumeRole" - ], - "Effect": "Allow", - "Principal": { - "Service": [ - "lambda.amazonaws.com" - ] - } + "Version": "2012-10-17", + "Statement": [{ + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] } - ] + }] } } - }, + }, "KinesisFunctionMyKinesisStream": { - "Type": "AWS::Lambda::EventSourceMapping", + "Type": "AWS::Lambda::EventSourceMapping", "Properties": { - "BatchSize": 100, + "BatchSize": 100, "EventSourceArn": "arn:aws:kinesis:us-west-2:012345678901:stream/my-stream", "FunctionName": { "Ref": "KinesisFunction" - }, + }, "StartingPosition": "TRIM_HORIZON" } - }, + }, "KinesisFunctionRole": { - "Type": "AWS::IAM::Role", + "Type": "AWS::IAM::Role", "Properties": { "ManagedPolicyArns": [ "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaKinesisExecutionRole" - ], + ], "AssumeRolePolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Action": [ - "sts:AssumeRole" - ], - "Effect": "Allow", - "Principal": { - "Service": [ - "lambda.amazonaws.com" - ] - } + "Version": "2012-10-17", + "Statement": [{ + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] } - ] + }] } } } diff --git a/tests/translator/output/error_missing_queue.json b/tests/translator/output/error_missing_queue.json new file mode 100644 index 000000000..f92bbf80e --- /dev/null +++ b/tests/translator/output/error_missing_queue.json @@ -0,0 +1,6 @@ +{ + "errors": [{ + "errorMessage": "Resource with id [SQSFunction] is invalid. Event with id [MySqsQueue] is invalid. No Queue (for SQS) or Stream (for Kinesis or DynamoDB) provided." + }], + "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [SQSFunction] is invalid. Event with id [MySqsQueue] is invalid. No Queue (for SQS) or Stream (for Kinesis or DynamoDB) provided." +} \ No newline at end of file diff --git a/tests/translator/output/error_missing_startingposition.json b/tests/translator/output/error_missing_startingposition.json new file mode 100644 index 000000000..4e3da968c --- /dev/null +++ b/tests/translator/output/error_missing_startingposition.json @@ -0,0 +1,6 @@ +{ + "errors": [{ + "errorMessage": "Resource with id [KinesisFunction] is invalid. Event with id [MyKinesisStream] is invalid. StartingPosition is required for Kinesis and DynamoDB." + }], + "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [KinesisFunction] is invalid. Event with id [MyKinesisStream] is invalid. StartingPosition is required for Kinesis and DynamoDB." +} \ No newline at end of file diff --git a/tests/translator/output/error_missing_stream.json b/tests/translator/output/error_missing_stream.json new file mode 100644 index 000000000..d8925a773 --- /dev/null +++ b/tests/translator/output/error_missing_stream.json @@ -0,0 +1,6 @@ +{ + "errors": [{ + "errorMessage": "Resource with id [DynamoDBFunction] is invalid. Event with id [MyDDBStream] is invalid. No Queue (for SQS) or Stream (for Kinesis or DynamoDB) provided." + }], + "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 1. Resource with id [DynamoDBFunction] is invalid. Event with id [MyDDBStream] is invalid. No Queue (for SQS) or Stream (for Kinesis or DynamoDB) provided." +} \ No newline at end of file diff --git a/tests/translator/output/implicit_api.json b/tests/translator/output/implicit_api.json index dcc83e827..36385f318 100644 --- a/tests/translator/output/implicit_api.json +++ b/tests/translator/output/implicit_api.json @@ -98,7 +98,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/*", + "arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/*", { "__Stage__": "Prod", "__ApiId__": { @@ -200,7 +200,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/*", + "arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/*", { "__Stage__": "*", "__ApiId__": { diff --git a/tests/translator/output/implicit_api_with_serverless_rest_api_resource.json b/tests/translator/output/implicit_api_with_serverless_rest_api_resource.json index dcc83e827..36385f318 100644 --- a/tests/translator/output/implicit_api_with_serverless_rest_api_resource.json +++ b/tests/translator/output/implicit_api_with_serverless_rest_api_resource.json @@ -98,7 +98,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/*", + "arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/*", { "__Stage__": "Prod", "__ApiId__": { @@ -200,7 +200,7 @@ }, "SourceArn": { "Fn::Sub": [ - "arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/ANY/*", + "arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${__ApiId__}/${__Stage__}/*/*", { "__Stage__": "*", "__ApiId__": { diff --git a/tests/translator/output/sqs.json b/tests/translator/output/sqs.json new file mode 100644 index 000000000..0ca3a1860 --- /dev/null +++ b/tests/translator/output/sqs.json @@ -0,0 +1,58 @@ +{ + "Resources": { + "SQSFunctionRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "ManagedPolicyArns": [ + "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + "arn:aws:iam::aws:policy/service-role/AWSLambdaSQSExecutionRole" + ], + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] + } + }] + } + } + }, + "SQSFunction": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": "sam-demo-bucket", + "S3Key": "queues.zip" + }, + "Handler": "queue.sqs_handler", + "Role": { + "Fn::GetAtt": [ + "SQSFunctionRole", + "Arn" + ] + }, + "Runtime": "python2.7", + "Tags": [{ + "Value": "SAM", + "Key": "lambda:createdBy" + }] + } + }, + "SQSFunctionMySqsQueue": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties": { + "BatchSize": 10, + "EventSourceArn": "arn:aws:sqs:us-west-2:012345678901:my-queue", + "FunctionName": { + "Ref": "SQSFunction" + } + } + } + } +} \ No newline at end of file diff --git a/tests/translator/output/streams.json b/tests/translator/output/streams.json index b814232e7..b8847e458 100644 --- a/tests/translator/output/streams.json +++ b/tests/translator/output/streams.json @@ -1,120 +1,112 @@ { "Resources": { "DynamoDBFunction": { - "Type": "AWS::Lambda::Function", + "Type": "AWS::Lambda::Function", "Properties": { "Code": { "S3Bucket": "sam-demo-bucket", "S3Key": "streams.zip" - }, - "Handler": "stream.ddb_handler", + }, + "Handler": "stream.ddb_handler", "Role": { "Fn::GetAtt": [ - "DynamoDBFunctionRole", + "DynamoDBFunctionRole", "Arn" ] - }, - "Runtime": "python2.7", - "Tags": [ - { - "Value": "SAM", - "Key": "lambda:createdBy" - } - ] + }, + "Runtime": "python2.7", + "Tags": [{ + "Value": "SAM", + "Key": "lambda:createdBy" + }] } - }, + }, "KinesisFunction": { - "Type": "AWS::Lambda::Function", + "Type": "AWS::Lambda::Function", "Properties": { "Code": { "S3Bucket": "sam-demo-bucket", "S3Key": "streams.zip" - }, - "Handler": "stream.kinesis_handler", + }, + "Handler": "stream.kinesis_handler", "Role": { "Fn::GetAtt": [ - "KinesisFunctionRole", + "KinesisFunctionRole", "Arn" ] - }, - "Runtime": "python2.7", - "Tags": [ - { - "Value": "SAM", - "Key": "lambda:createdBy" - } - ] + }, + "Runtime": "python2.7", + "Tags": [{ + "Value": "SAM", + "Key": "lambda:createdBy" + }] } - }, + }, "DynamoDBFunctionMyDDBStream": { - "Type": "AWS::Lambda::EventSourceMapping", + "Type": "AWS::Lambda::EventSourceMapping", "Properties": { - "BatchSize": 200, - "EventSourceArn": "arn:aws:dynamodb:us-west-2:012345678901:table/TestTable/stream/2015-05-11T21:21:33.291", + "BatchSize": 200, + "EventSourceArn": "arn:aws:dynamodb:us-west-2:012345678901:table/TestTable/stream/2015-05-11T21:21:33.291", "FunctionName": { "Ref": "DynamoDBFunction" - }, + }, "StartingPosition": "LATEST" } - }, + }, "DynamoDBFunctionRole": { - "Type": "AWS::IAM::Role", + "Type": "AWS::IAM::Role", "Properties": { "ManagedPolicyArns": [ - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", "arn:aws:iam::aws:policy/service-role/AWSLambdaDynamoDBExecutionRole" - ], + ], "AssumeRolePolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Action": [ - "sts:AssumeRole" - ], - "Effect": "Allow", - "Principal": { - "Service": [ - "lambda.amazonaws.com" - ] - } + "Version": "2012-10-17", + "Statement": [{ + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] } - ] + }] } } - }, + }, "KinesisFunctionMyKinesisStream": { - "Type": "AWS::Lambda::EventSourceMapping", + "Type": "AWS::Lambda::EventSourceMapping", "Properties": { - "BatchSize": 100, - "EventSourceArn": "arn:aws:kinesis:us-west-2:012345678901:stream/my-stream", + "BatchSize": 100, + "EventSourceArn": "arn:aws:kinesis:us-west-2:012345678901:stream/my-stream", "FunctionName": { "Ref": "KinesisFunction" - }, + }, "StartingPosition": "TRIM_HORIZON" } - }, + }, "KinesisFunctionRole": { - "Type": "AWS::IAM::Role", + "Type": "AWS::IAM::Role", "Properties": { "ManagedPolicyArns": [ - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", "arn:aws:iam::aws:policy/service-role/AWSLambdaKinesisExecutionRole" - ], + ], "AssumeRolePolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Action": [ - "sts:AssumeRole" - ], - "Effect": "Allow", - "Principal": { - "Service": [ - "lambda.amazonaws.com" - ] - } + "Version": "2012-10-17", + "Statement": [{ + "Action": [ + "sts:AssumeRole" + ], + "Effect": "Allow", + "Principal": { + "Service": [ + "lambda.amazonaws.com" + ] } - ] + }] } } } diff --git a/tests/translator/test_translator.py b/tests/translator/test_translator.py index 664dba5be..40810b5b6 100644 --- a/tests/translator/test_translator.py +++ b/tests/translator/test_translator.py @@ -11,7 +11,7 @@ from samtranslator.public.plugins import BasePlugin from tests.translator.helpers import get_template_parameter_values -from tests.translator.yaml_helper import yaml_parse +from samtranslator.yaml_helper import yaml_parse from parameterized import parameterized, param import pytest @@ -46,6 +46,7 @@ class TestTranslatorEndToEnd(TestCase): 'cloudwatch_logs_with_ref', 'cloudwatchlog', 'streams', + 'sqs', 'simpletable', 'simpletable_with_sse', 'implicit_api', @@ -156,6 +157,9 @@ def test_transform_success(self, testcase, partition_with_region): 'error_function_with_deployment_preference_missing_alias', 'error_function_with_invalid_deployment_preference_hook_property', 'error_invalid_logical_id', + 'error_missing_queue', + 'error_missing_startingposition', + 'error_missing_stream', 'error_multiple_resource_errors', 'error_s3_not_in_template', 'error_table_invalid_attributetype', diff --git a/tests/translator/validator/test_validator.py b/tests/translator/validator/test_validator.py index a7dd8a0c0..4340a3cd9 100644 --- a/tests/translator/validator/test_validator.py +++ b/tests/translator/validator/test_validator.py @@ -1,7 +1,7 @@ import os.path import pytest from unittest import TestCase -from tests.translator.yaml_helper import yaml_parse +from samtranslator.yaml_helper import yaml_parse from samtranslator.validator.validator import SamTemplateValidator input_folder = 'tests/translator/input' @@ -12,6 +12,7 @@ 'cloudwatch_logs_with_ref', 'cloudwatchlog', 'streams', + 'sqs', 'simpletable', 'simpletable_with_sse', 'implicit_api', diff --git a/versions/2016-10-31.md b/versions/2016-10-31.md index 4c742eae6..f29251d4a 100644 --- a/versions/2016-10-31.md +++ b/versions/2016-10-31.md @@ -279,6 +279,7 @@ Properties: - [SNS](#sns) - [Kinesis](#kinesis) - [DynamoDB](#dynamodb) + - [SQS](#sqs) - [Api](#api) - [Schedule](#schedule) - [CloudWatchEvent](#cloudwatchevent) @@ -371,6 +372,26 @@ Properties: BatchSize: 10 ``` +#### SQS + +The object describing an event source with type `SQS`. + +##### Properties + +Property Name | Type | Description +---|:---:|--- +Queue | `string` | **Required.** ARN of the SQS queue. +BatchSize | `integer` | Maximum number of messages to process per function invocation. + +##### Example: SQS event source object + +```yaml +Type: SQS +Properties: + Queue: arn:aws:sqs:us-west-2:012345678901:my-queue # NOTE: FIFO SQS Queues are not yet supported + BatchSize: 10 +``` + #### Api The object describing an event source with type `Api`.