Fix different python runtimes errors

Author: Michal Ploski

.github/workflows/run-e2e-tests.yml

@@ -6,19 +6,13 @@ env:
   E2E_TESTS_PATH: tests/e2e/
 jobs:
   run:
-    #########################
-    # Force Github action to run only a single job at a time (based on the group name)
-    # This is to prevent "race-condition" in building e2e tests infrastructure
-    #########################
-    concurrency:
-      group: e2e-tests
     runs-on: ubuntu-latest
     permissions:
       id-token: write # needed to interact with GitHub's OIDC Token endpoint.
       contents: read
     strategy:
       matrix:
-        version: ["3.8", "3.9", "3.10"]
+        version: ["3.6", "3.7", "3.8", "3.9"]
     steps:
       - name: "Checkout"
         uses: actions/checkout@v3

tests/e2e/conftest.py

@@ -1,18 +1,26 @@
 import datetime
+import sys
 import uuid
-from typing import Generator, TypedDict
+
+# We only need typing_extensions for python versions <3.8
+if sys.version_info >= (3, 8):
+    from typing import TypedDict
+else:
+    from typing_extensions import TypedDict
+
+from typing import Dict, Generator
 
 import pytest
 from e2e.utils import helpers, infrastructure
 
 
 class LambdaConfig(TypedDict):
     parameters: dict
-    environment_variables: dict[str, str]
+    environment_variables: Dict[str, str]
 
 
 class LambdaExecution(TypedDict):
-    arns: dict[str, str]
+    arns: Dict[str, str]
     execution_time: datetime.datetime
 
 

tests/e2e/logger/test_logger.py

@@ -1,5 +1,3 @@
-from typing import TypedDict
-
 import boto3
 import pytest
 

tests/e2e/utils/infrastructure.py

@@ -4,7 +4,7 @@
 import zipfile
 from enum import Enum
 from pathlib import Path
-from typing import Any
+from typing import Dict, List
 
 import boto3
 import yaml
@@ -29,13 +29,13 @@ def __init__(self, stack_name: str, handlers_dir: str, config: dict, environment
         self.cf_client = session.client("cloudformation")
         self.s3_resource = session.resource("s3")
         self.account_id = session.client("sts").get_caller_identity()["Account"]
-        self.region = boto3.Session().region_name
+        self.region = session.region_name
         self.stack_name = stack_name
         self.handlers_dir = handlers_dir
         self.config = config
         self.environment_variables = environment_variables
 
-    def deploy(self) -> dict[str, str]:
+    def deploy(self) -> Dict[str, str]:
         handlers = self._find_files(directory=self.handlers_dir, only_py=True)
         template, asset_root_dir = self.prepare_stack(
             handlers=handlers,
@@ -55,7 +55,7 @@ def delete(self):
 
     # Create CDK cloud assembly code
     def prepare_stack(
-        self, handlers: list[str], handlers_dir: str, stack_name: str, environment_variables: dict, **config: dict
+        self, handlers: List[str], handlers_dir: str, stack_name: str, environment_variables: dict, **config: dict
     ):
         integration_test_app = App()
         stack = Stack(integration_test_app, stack_name)
@@ -99,17 +99,16 @@ def _find_files(self, directory: str, only_py: bool = False) -> list:
                     file_paths.append(os.path.join(root, filename))
         return file_paths
 
-    def _create_layer(self, stack):
-        output_dir = Path(AssetStaging.BUNDLING_OUTPUT_DIR, "python")
-        input_dir = Path(AssetStaging.BUNDLING_INPUT_DIR, "aws_lambda_powertools")
+    def _create_layer(self, stack: Stack):
+        output_dir = Path(str(AssetStaging.BUNDLING_OUTPUT_DIR), "python")
+        input_dir = Path(str(AssetStaging.BUNDLING_INPUT_DIR), "aws_lambda_powertools")
         powertools_layer = LayerVersion(
             stack,
             "aws-lambda-powertools",
             layer_version_name="aws-lambda-powertools",
             compatible_runtimes=[PythonVersion[PYTHON_RUNTIME_VERSION].value["runtime"]],
             code=Code.from_asset(
                 path=".",
-                exclude=["*.pyc"],
                 bundling=BundlingOptions(
                     image=DockerImage.from_build(
                         str(Path(__file__).parent),
@@ -118,12 +117,14 @@ def _create_layer(self, stack):
                     command=[
                         "bash",
                         "-c",
-                        f"poetry export --with-credentials --format requirements.txt --output requirements.txt && pip install -r requirements.txt -t {output_dir} && cp -R {input_dir} {output_dir}",
+                        f"poetry export --with-credentials --format requirements.txt --output /tmp/requirements.txt &&\
+                            pip install -r /tmp/requirements.txt -t {output_dir} &&\
+                            cp -R {input_dir} {output_dir} &&\
+                            find {output_dir}/ -regex '^.*\\(__pycache__\\|\\.py[co]\\)$' -delete",
                     ],
                 ),
             ),
         )
-
         return powertools_layer
 
     def _upload_assets(self, template: dict, asset_root_dir: str):
@@ -146,16 +147,16 @@ def _upload_assets(self, template: dict, asset_root_dir: str):
             buf.seek(0)
             self.s3_client.upload_fileobj(Fileobj=buf, Bucket=bucket, Key=s3_key)
 
-    def _deploy_stack(self, stack_name: str, template: Any):
+    def _deploy_stack(self, stack_name: str, template: dict):
         response = self.cf_client.create_stack(
             StackName=stack_name,
             TemplateBody=yaml.dump(template),
             TimeoutInMinutes=10,
-            OnFailure="DO_NOTHING",
+            OnFailure="ROLLBACK",
             Capabilities=["CAPABILITY_IAM"],
         )
         waiter = self.cf_client.get_waiter("stack_create_complete")
-        waiter.wait(StackName=stack_name, WaiterConfig={"Delay": 2, "MaxAttempts": 50})
+        waiter.wait(StackName=stack_name, WaiterConfig={"Delay": 10, "MaxAttempts": 50})
         response = self.cf_client.describe_stacks(StackName=stack_name)
         return response