Merge pull request #2969 from pradhapanr/feat-appsync

chore: rebase feat-appsync

Author: ssenchenko

.cfnlintrc.yaml

@@ -57,6 +57,7 @@ ignore_templates:
   - tests/translator/output/**/connector_sfn_to_function.json
   - tests/translator/output/**/connector_sns_to_function.json
   - tests/translator/output/**/connector_table_to_function.json
+  - tests/translator/output/**/documentdb_with_intrinsics.json  # TODO: remove once DocumentDDB is available
   - tests/translator/output/**/eventbridgerule_with_dlq.json
   - tests/translator/output/**/function_event_conditions.json
   - tests/translator/output/**/function_with_alias_and_code_sha256.json
@@ -77,6 +78,8 @@ ignore_templates:
   - tests/translator/output/**/function_with_deployment_preference_multiple_combinations_conditions_without_passthrough.json
   - tests/translator/output/**/function_with_deployment_preference_passthrough_condition_with_supported_intrinsics.json
   - tests/translator/output/**/function_with_dlq.json
+  - tests/translator/output/**/function_with_documentdb_with_kms.json  # TODO: remove once DocumentDDB is available
+  - tests/translator/output/**/function_with_documentdb.json  # TODO: remove once DocumentDDB is available
   - tests/translator/output/**/function_with_event_dest.json
   - tests/translator/output/**/function_with_event_dest_basic.json
   - tests/translator/output/**/function_with_event_dest_conditional.json
@@ -117,6 +120,7 @@ ignore_templates:
   - tests/translator/output/**/state_machine_with_schedule_dlq_retry_policy.json
   - tests/translator/output/**/globals_for_function.json  # RuntimeManagementConfig
   - tests/translator/output/**/function_with_runtime_config.json  # RuntimeManagementConfig
+  - tests/translator/output/**/managed_policies_minimal.json  # Intentionally has non-existent managed policy name
 ignore_checks:
   - E2531 # Deprecated runtime; not relevant for transform tests
   - W2531 # EOL runtime; not relevant for transform tests

.coveragerc

@@ -3,6 +3,7 @@ branch = True
 omit =
     # Schema tested by `make check-black`
     samtranslator/schema/*
+    samtranslator/internal/schema_source/*
 [report]
 exclude_lines =
     pragma: no cover

DEVELOPMENT_GUIDE.md

@@ -167,7 +167,7 @@ Integration tests are covered in detail in the [INTEGRATION_TESTS.md file](INTEG
 
    For new properties of SAM resources, use [`Property`](https://github.com/aws/serverless-application-model/blob/c5830b63857f52e540fec13b29f029458edc539a/samtranslator/model/__init__.py#L36-L45) or [`PassThroughProperty`](https://github.com/aws/serverless-application-model/blob/dd79f535500158baa8e367f081d6a12113497e45/samtranslator/model/__init__.py#L48-L56) instead of [`PropertyType`](https://github.com/aws/serverless-application-model/blob/c39c2807bbf327255de8abed8b8150b18c60f053/samtranslator/model/__init__.py#L13-L33). This avoids [sneaky bugs](https://github.com/aws/serverless-application-model/pull/2495#discussion_r976715242) and ensures valid templates do not cause transform failures.
 
-   For new properties of CloudFormation resources, use `GeneratedProperty`. It performs no runtime validation, reducing the risk of valid values causing transform failures.
+   For new properties of CloudFormation resources, use [`GeneratedProperty`](https://github.com/aws/serverless-application-model/blob/79452f69bc1fcf918b8625c2f9005c74ab874801/samtranslator/model/__init__.py#L74-L82). It performs no runtime validation, reducing the risk of valid values causing transform failures.
 
 Code conventions
 ----------------

MANIFEST.in

@@ -8,6 +8,8 @@ recursive-include samtranslator/validator/sam_schema *.json
 include samtranslator/policy_templates_data/policy_templates.json
 include samtranslator/policy_templates_data/schema.json
 include samtranslator/model/connector_profiles/profiles.json
+include samtranslator/internal/data/aws_managed_policies.json
+include samtranslator/internal/schema_source/sam-docs.json
 include README.md
 include THIRD_PARTY_LICENSES
 

Makefile

@@ -26,7 +26,7 @@ black:
 black-check:
 	# Checking latest schema was generated (run `make schema` if this fails)
 	mkdir -p .tmp
-	python -m schema_source.schema --sam-schema .tmp/sam.schema.json --cfn-schema schema_source/cloudformation.schema.json --unified-schema .tmp/schema.json
+	python -m samtranslator.internal.schema_source.schema --sam-schema .tmp/sam.schema.json --cfn-schema schema_source/cloudformation.schema.json --unified-schema .tmp/schema.json
 	diff -u schema_source/sam.schema.json .tmp/sam.schema.json
 	diff -u samtranslator/schema/schema.json .tmp/schema.json
 	black --check setup.py samtranslator tests integration bin schema_source
@@ -60,14 +60,14 @@ fetch-schema-data:
 
 update-schema-data:
 	# Parse docs
-	bin/parse_docs.py .tmp/aws-sam-developer-guide/doc_source > schema_source/docs.json
+	bin/parse_docs.py .tmp/aws-sam-developer-guide/doc_source > samtranslator/internal/schema_source/sam-docs.json
 	bin/parse_docs.py --cfn .tmp/aws-cloudformation-user-guide/doc_source > schema_source/cloudformation-docs.json
 
 	# Add CloudFormation docs to CloudFormation schema
 	python bin/add_docs_cfn_schema.py --schema .tmp/cloudformation.schema.json --docs schema_source/cloudformation-docs.json > schema_source/cloudformation.schema.json
 
 schema:
-	python -m schema_source.schema --sam-schema schema_source/sam.schema.json --cfn-schema schema_source/cloudformation.schema.json --unified-schema samtranslator/schema/schema.json
+	python -m samtranslator.internal.schema_source.schema --sam-schema schema_source/sam.schema.json --cfn-schema schema_source/cloudformation.schema.json --unified-schema samtranslator/schema/schema.json
 
 # Update all schema data and schemas
 schema-all: fetch-schema-data update-schema-data schema

bin/parse_docs.py

@@ -89,10 +89,10 @@ def main() -> None:
         if args.cfn and not re.match(r"^\w+::\w+::\w+( \w+)?$", title):
             continue
         page = title if args.cfn else path.stem
-        for name, description in parse(text):
+        for name, raw_description in parse(text):
             if page not in props:
                 props[page] = {}
-            description = remove_first_line(description)  # Remove property name; already in the schema title
+            description = remove_first_line(raw_description)  # Remove property name; already in the schema title
             description = fix_markdown_code_link(description)
             prefix = (
                 "https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/"

bin/sam-translate.py

@@ -106,7 +106,7 @@ def transform_template(input_file_path, output_file_path):  # type: ignore[no-un
         sam_template = yaml_parse(f)  # type: ignore[no-untyped-call]
 
     try:
-        cloud_formation_template = transform(sam_template, {}, ManagedPolicyLoader(iam_client))  # type: ignore[no-untyped-call]
+        cloud_formation_template = transform(sam_template, {}, ManagedPolicyLoader(iam_client))
         cloud_formation_template_prettified = json.dumps(cloud_formation_template, indent=1)
 
         with open(output_file_path, "w") as f:

docs/globals.rst

@@ -81,6 +81,7 @@ Currently, the following resources and properties are being supported:
       Auth:
       Name:
       DefinitionUri:
+      MergeDefinitions:
       CacheClusterEnabled:
       CacheClusterSize:
       Variables:

integration/combination/test_connectors.py

@@ -26,6 +26,7 @@ def tearDown(self):
 
     @parameterized.expand(
         [
+            ("combination/connector_appsync_to_table",),
             ("combination/connector_function_to_function",),
             ("combination/connector_restapi_to_function",),
             ("combination/connector_httpapi_to_function",),

integration/config/service_names.py

@@ -35,3 +35,4 @@
 EVENT_INVOKE_CONFIG = "EventInvokeConfig"
 EPHEMERAL_STORAGE = "EphemeralStorage"
 API_KEY = "ApiKey"
+APP_SYNC = "AppSync"

integration/helpers/resource.py

@@ -5,6 +5,7 @@
 from typing import Any, Callable, Dict, Set
 
 from integration.config.service_names import (
+    APP_SYNC,
     DYNAMO_DB,
     HTTP_API,
     LOCATION,
@@ -223,6 +224,7 @@ def _resource_using_s3_events(resource: Dict[str, Any]) -> bool:
         _resource_using_s3_events(resource) for resource in template_dict.get("Resources", {}).values()
     ),
     LOCATION: lambda template_dict, cfn_resource_types: "AWS::Location::PlaceIndex" in cfn_resource_types,
+    APP_SYNC: lambda template_dict, cfn_resource_types: "AWS::AppSync::GraphQLApi" in cfn_resource_types,
 }
 
 

integration/resources/expected/combination/connector_appsync_to_table.json

@@ -0,0 +1,38 @@
+[
+  {
+    "LogicalResourceId": "NotesTable",
+    "ResourceType": "AWS::DynamoDB::Table"
+  },
+  {
+    "LogicalResourceId": "DynamoDBRole",
+    "ResourceType": "AWS::IAM::Role"
+  },
+  {
+    "LogicalResourceId": "AppSyncApi",
+    "ResourceType": "AWS::AppSync::GraphQLApi"
+  },
+  {
+    "LogicalResourceId": "ApiKey",
+    "ResourceType": "AWS::AppSync::ApiKey"
+  },
+  {
+    "LogicalResourceId": "ApiSchema",
+    "ResourceType": "AWS::AppSync::GraphQLSchema"
+  },
+  {
+    "LogicalResourceId": "NotesTableDataSource",
+    "ResourceType": "AWS::AppSync::DataSource"
+  },
+  {
+    "LogicalResourceId": "TriggerFunction",
+    "ResourceType": "AWS::Lambda::Function"
+  },
+  {
+    "LogicalResourceId": "TriggerFunctionRole",
+    "ResourceType": "AWS::IAM::Role"
+  },
+  {
+    "LogicalResourceId": "DataSourceToTableConnectorPolicy",
+    "ResourceType": "AWS::IAM::ManagedPolicy"
+  }
+]

integration/resources/templates/combination/connector_appsync_to_table.yaml

@@ -0,0 +1,203 @@
+Resources:
+  NotesTable:
+    Type: AWS::Serverless::SimpleTable
+    Properties:
+      PrimaryKey:
+        Name: NoteId
+        Type: String
+
+  DynamoDBRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+        - Effect: Allow
+          Action:
+          - sts:AssumeRole
+          Principal:
+            Service:
+            - appsync.amazonaws.com
+
+  AppSyncApi:
+    Type: AWS::AppSync::GraphQLApi
+    Properties:
+      AuthenticationType: API_KEY
+      Name: AppSyncApi
+
+  ApiKey:
+    Type: AWS::AppSync::ApiKey
+    Properties:
+      ApiId: !GetAtt AppSyncApi.ApiId
+
+  ApiSchema:
+    Type: AWS::AppSync::GraphQLSchema
+    Properties:
+      ApiId: !GetAtt AppSyncApi.ApiId
+      Definition: |
+        type Note {
+          NoteId: ID!
+          title: String
+          content: String
+        }
+        type Query {
+          getNote(NoteId: ID!): Note
+        }
+        type Mutation {
+          saveNote(NoteId: ID!, title: String!, content: String!): Note!
+        }
+        type Schema {
+          query: Query
+          mutation: Mutation
+        }
+
+  NotesTableDataSource:
+    Type: AWS::AppSync::DataSource
+    Properties:
+      ApiId: !GetAtt AppSyncApi.ApiId
+      Name: NotesTableDataSource
+      Type: AMAZON_DYNAMODB
+      ServiceRoleArn: !GetAtt DynamoDBRole.Arn
+      DynamoDBConfig:
+        TableName: !Ref NotesTable
+        AwsRegion: !Sub ${AWS::Region}
+
+  DataSourceToTableConnector:
+    Type: AWS::Serverless::Connector
+    Properties:
+      Source:
+        Id: NotesTableDataSource
+      Destination:
+        Id: NotesTable
+      Permissions:
+      - Read
+      - Write
+
+  TriggerFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      Environment:
+        Variables:
+          API_KEY: !GetAtt ApiKey.ApiKey
+          GRAPHQL_URL: !GetAtt AppSyncApi.GraphQLUrl
+      Runtime: nodejs14.x
+      Handler: index.handler
+      InlineCode: |
+        const https = require("https");
+
+        exports.handler = async (_) => {
+          const queries = {
+            getNote: /* GraphQL */ `
+              query {
+                getNote(NoteId: "1") {
+                  title
+                  content
+                }
+              }
+            `,
+            saveNote: /* GraphQL */ `
+              mutation {
+                saveNote(content: "some note", NoteId: "1", title: "1st note") {
+                  title
+                  content
+                }
+              }
+            `,
+          };
+
+          const fetch = async (url, options) =>
+            new Promise((resolve, reject) => {
+              const req = https.request(url, options, (res) => {
+                const body = [];
+                res.on("data", (chunk) => body.push(chunk));
+                res.on("end", () => {
+                  const resString = Buffer.concat(body).toString();
+                  resolve(resString);
+                });
+              });
+
+              req.on("error", (err) => {
+                reject(err);
+              });
+
+              req.on("timeout", () => {
+                req.destroy();
+                reject(new Error("Request time out"));
+              });
+
+              req.write(options.body);
+              req.end();
+            });
+
+          const makeRequest = async (queryName) => {
+            const options = {
+              method: "POST",
+              headers: {
+                "x-api-key": process.env.API_KEY,
+              },
+              body: JSON.stringify({ query: queries[queryName] }),
+              timeout: 10000, // ms
+            };
+
+            let statusCode;
+            let body;
+            let response;
+
+            try {
+              response = await fetch(process.env.GRAPHQL_URL, options);
+              body = JSON.parse(response);
+              const data = body.data?.[queryName];
+              const hasNoErrors = body.errors === undefined;
+              const allFieldsAreSet =
+                data?.title === "1st note" && data?.content === "some note";
+              statusCode = hasNoErrors && allFieldsAreSet ? 200 : 400;
+              if (hasNoErrors) {
+                body = body.data;
+              } else {
+                body = {
+                  [queryName]: {
+                    errors: body.errors,
+                  },
+                };
+              }
+            } catch (error) {
+              statusCode = 400;
+              body = {
+                [queryName]: {
+                  errors: [
+                    {
+                      status: response.status,
+                      message: error.message,
+                      stack: error.stack,
+                    },
+                  ],
+                },
+              };
+            }
+            return {
+              statusCode,
+              body,
+            };
+          };
+
+          let response = await makeRequest("saveNote");
+          if (response.statusCode !== 200) {
+            return {
+              StatusCode: response.statusCode,
+              Body: response.body,
+            };
+          }
+          let body = response.body;
+
+          response = await makeRequest("getNote");
+          body = { ...body, ...response.body };
+
+          return {
+            StatusCode: response.statusCode,
+            Body: body,
+          };
+        };
+
+
+Metadata:
+  SamTransformTest: true

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.black]
 line-length = 120
-target_version = ['py37', 'py36', 'py38']
+target_version = ['py37', 'py38', 'py39', 'py310']
 exclude = '''
 
 (

requirements/base.txt

@@ -1,3 +1,6 @@
 boto3>=1.19.5,==1.*
 jsonschema<5,>=3.2  # TODO: evaluate risk of removing jsonschema 3.x support
-typing_extensions~=4.4.0 # 3.7 doesn't have Literal
+typing_extensions>=4.4,<5 # 3.7 doesn't have Literal
+
+# resource validation & schema generation
+pydantic~=1.8