Refactor code to use common libs

Author: Michal Ploski

poetry.lock

@@ -31,6 +31,7 @@ python = "^3.6.2"
 # 2022-04-21: jmespath was removed, to be re-added once we drop python 3.6.
 # issue #1148
 aws-cdk-lib = "^2.23.0"
+"aws-cdk.aws-lambda-python-alpha" = "2.23.0-alpha.0"
 bandit = "^1.7.1"
 black = "^21.12b0"
 coverage = {extras = ["toml"], version = "^6.2"}

pyproject.toml

@@ -1,95 +1,31 @@
+import datetime
 import uuid
 
-import boto3
 import pytest
-import yaml
-from aws_cdk import App, CfnOutput, RemovalPolicy, Stack, aws_lambda_python_alpha, aws_logs
-from aws_cdk.aws_lambda import Code, Function, Runtime, Tracing
 
-from . import utils
+from tests.e2e.utils import helpers, infrastructure
 
 
-def transform_output(outputs):
-    return {output["OutputKey"]: output["OutputValue"] for output in outputs if output["OutputKey"]}
-
-
-# Create CDK cloud assembly code
-def prepare_infrastructure(handlers_name, handlers_dir, stack_name, environment_variables, **config):
-    integration_test_app = App()
-    stack = Stack(integration_test_app, stack_name)
-    powertools_layer = aws_lambda_python_alpha.PythonLayerVersion(
-        stack,
-        "aws-lambda-powertools",
-        layer_version_name="aws-lambda-powertools",
-        entry=".",
-        compatible_runtimes=[Runtime.PYTHON_3_9],
-    )
-    code = Code.from_asset(handlers_dir)
-
-    for filename in handlers_name:
-
-        function_python = Function(
-            stack,
-            f"{filename}-lambda",
-            runtime=Runtime.PYTHON_3_9,
-            code=code,
-            handler=f"{filename}.lambda_handler",
-            layers=[powertools_layer],
-            environment=environment_variables,
-            tracing=Tracing.ACTIVE if config.get("tracing") == "ACTIVE" else Tracing.DISABLED,
-        )
-
-        aws_logs.LogGroup(
-            stack,
-            f"{filename}-lg",
-            log_group_name=f"/aws/lambda/{function_python.function_name}",
-            retention=aws_logs.RetentionDays.ONE_DAY,
-            removal_policy=RemovalPolicy.DESTROY,
-        )
-        CfnOutput(stack, f"{filename}_arn", value=function_python.function_arn)
-    return (
-        integration_test_app.synth().get_stack_by_name(stack_name).template,
-        integration_test_app.synth().directory,
-        integration_test_app.synth().artifacts,
-    )
-
-
-def deploy_infrastructure(template, asset_root_dir, stack_name, client):
-
-    utils.upload_assets(template, asset_root_dir)
-
-    response = client.create_stack(
-        StackName=stack_name,
-        TemplateBody=yaml.dump(template),
-        TimeoutInMinutes=10,
-        OnFailure="DO_NOTHING",
-        Capabilities=["CAPABILITY_IAM"],
-    )
-    waiter = client.get_waiter("stack_create_complete")
-    waiter.wait(StackName=stack_name, WaiterConfig={"Delay": 2, "MaxAttempts": 50})
-    response = client.describe_stacks(StackName=stack_name)
-    return response["Stacks"][0]["Outputs"]
-
-
-@pytest.fixture(scope="session")
-def deploy():
-    cf_client = boto3.Session().client("cloudformation")
+@pytest.fixture(scope="module")
+def execute_lambda(config, request):
     stack_name = f"test-lambda-{uuid.uuid4()}"
+    test_dir = request.fspath.dirname
+    handlers_dir = f"{test_dir}/handlers/"
+
+    infra = infrastructure.Infrastructure(
+        stack_name=stack_name,
+        handlers_dir=handlers_dir,
+        config=config["parameters"],
+        environment_variables=config["environment_variables"],
+    )
 
-    def deploy(handlers_name, handlers_dir, environment_variables, **config):
+    lambda_arns = infra.deploy()
+    execution_time = datetime.datetime.utcnow()
 
-        template, asset_root_dir, artifact = prepare_infrastructure(
-            handlers_name=handlers_name,
-            handlers_dir=handlers_dir,
-            stack_name=stack_name,
-            environment_variables=environment_variables,
-            **config,
-        )
-        outputs = deploy_infrastructure(
-            template=template, asset_root_dir=asset_root_dir, stack_name=stack_name, client=cf_client
-        )
-        return transform_output(outputs)
+    for name, arn in lambda_arns.items():
+        helpers.trigger_lambda(lambda_arn=arn, client=infra.lambda_client)
+        print(f"lambda {name} triggered")
 
-    yield deploy
+    yield {"arns": lambda_arns, "execution_time": execution_time}
     # Ensure stack deletion is triggered at the end of the test session
-    cf_client.delete_stack(StackName=stack_name)
+    infra.delete()

tests/e2e/conftest.py

@@ -1,74 +1,72 @@
-import datetime
-import os
-from functools import lru_cache
-
 import boto3
 import pytest
 
-from .. import utils
-
-dirname = os.path.dirname(__file__)
+from ..utils import helpers
 
 
 @pytest.fixture(scope="module")
 def config():
-    return {"MESSAGE": "logger message test", "LOG_LEVEL": "INFO", "ADDITIONAL_KEY": "extra_info"}
+    return {
+        "parameters": {},
+        "environment_variables": {
+            "MESSAGE": "logger message test",
+            "LOG_LEVEL": "INFO",
+            "ADDITIONAL_KEY": "extra_info",
+        },
+    }
 
 
-@pytest.fixture(scope="module")
-def deploy_lambdas(deploy, config):
-    handlers_dir = f"{dirname}/handlers/"
-
-    lambda_arns = deploy(
-        handlers_name=utils.find_handlers(handlers_dir),
-        handlers_dir=handlers_dir,
-        environment_variables=config,
+@pytest.mark.e2e
+def test_basic_lambda_logs_visible(execute_lambda, config):
+    # GIVEN
+    lambda_arn = execute_lambda["arns"]["basichandlerarn"]
+    timestamp = int(execute_lambda["execution_time"].timestamp() * 1000)
+    cw_client = boto3.client("logs")
+
+    # WHEN
+    filtered_logs = helpers.get_logs(
+        lambda_function_name=lambda_arn.split(":")[-1], start_time=timestamp, log_client=cw_client
     )
 
-    for name, arn in lambda_arns.items():
-        utils.trigger_lambda(lambda_arn=arn)
-        print(f"lambda {name} triggered")
-    return lambda_arns
-
-
-@pytest.fixture(scope="module")
-def trigger_lambdas(deploy_lambdas):
-    for name, arn in deploy_lambdas.items():
-        utils.trigger_lambda(lambda_arn=arn)
-        print(f"lambda {name} triggered")
-
-
-@lru_cache(maxsize=10, typed=False)
-def fetch_logs(lambda_arn):
-    start_time = int(datetime.datetime.now().timestamp() * 1000)
-    result = utils.trigger_lambda(lambda_arn=lambda_arn)
-
-    filtered_logs = utils.get_logs(
-        start_time=start_time,
-        lambda_function_name=lambda_arn.split(":")[-1],
-        log_client=boto3.client("logs"),
+    # THEN
+    assert any(
+        log.message == config["environment_variables"]["MESSAGE"]
+        and log.level == config["environment_variables"]["LOG_LEVEL"]
+        for log in filtered_logs
     )
-    return filtered_logs
 
 
 @pytest.mark.e2e
-def test_basic_lambda_logs_visible(deploy_lambdas, config):
-
-    filtered_logs = fetch_logs(lambda_arn=deploy_lambdas["basichandlerarn"])
+def test_basic_lambda_no_debug_logs_visible(execute_lambda, config):
+    # GIVEN
+    lambda_arn = execute_lambda["arns"]["basichandlerarn"]
+    timestamp = int(execute_lambda["execution_time"].timestamp() * 1000)
+    cw_client = boto3.client("logs")
+
+    # WHEN
+    filtered_logs = helpers.get_logs(
+        lambda_function_name=lambda_arn.split(":")[-1], start_time=timestamp, log_client=cw_client
+    )
 
-    assert any(log.message == config["MESSAGE"] and log.level == config["LOG_LEVEL"] for log in filtered_logs)
+    # THEN
+    assert not any(
+        log.message == config["environment_variables"]["MESSAGE"] and log.level == "DEBUG" for log in filtered_logs
+    )
 
 
 @pytest.mark.e2e
-def test_basic_lambda_no_debug_logs_visible(deploy_lambdas, config):
-    filtered_logs = fetch_logs(lambda_arn=deploy_lambdas["basichandlerarn"])
-
-    assert not any(log.message == config["MESSAGE"] and log.level == "DEBUG" for log in filtered_logs)
-
+def test_basic_lambda_contextual_data_logged(execute_lambda):
+    # GIVEN
+    lambda_arn = execute_lambda["arns"]["basichandlerarn"]
+    timestamp = int(execute_lambda["execution_time"].timestamp() * 1000)
+    cw_client = boto3.client("logs")
+
+    # WHEN
+    filtered_logs = helpers.get_logs(
+        lambda_function_name=lambda_arn.split(":")[-1], start_time=timestamp, log_client=cw_client
+    )
 
-@pytest.mark.e2e
-def test_basic_lambda_contextual_data_logged(deploy_lambdas):
-    filtered_logs = fetch_logs(lambda_arn=deploy_lambdas["basichandlerarn"])
+    # THEN
     for log in filtered_logs:
         assert (
             log.xray_trace_id
@@ -81,26 +79,58 @@ def test_basic_lambda_contextual_data_logged(deploy_lambdas):
 
 
 @pytest.mark.e2e
-def test_basic_lambda_additional_key_persistence_basic_lambda(deploy_lambdas, config):
-    filtered_logs = fetch_logs(lambda_arn=deploy_lambdas["basichandlerarn"])
+def test_basic_lambda_additional_key_persistence_basic_lambda(execute_lambda, config):
+    # GIVEN
 
+    lambda_arn = execute_lambda["arns"]["basichandlerarn"]
+    timestamp = int(execute_lambda["execution_time"].timestamp() * 1000)
+    cw_client = boto3.client("logs")
+
+    # WHEN
+    filtered_logs = helpers.get_logs(
+        lambda_function_name=lambda_arn.split(":")[-1], start_time=timestamp, log_client=cw_client
+    )
+
+    # THEN
     assert any(
-        log.extra_info and log.message == config["MESSAGE"] and log.level == config["LOG_LEVEL"]
+        log.extra_info
+        and log.message == config["environment_variables"]["MESSAGE"]
+        and log.level == config["environment_variables"]["LOG_LEVEL"]
         for log in filtered_logs
     )
 
 
 @pytest.mark.e2e
-def test_basic_lambda_empty_event_logged(deploy_lambdas):
-    filtered_logs = fetch_logs(lambda_arn=deploy_lambdas["basichandlerarn"])
+def test_basic_lambda_empty_event_logged(execute_lambda):
+
+    # GIVEN
+    lambda_arn = execute_lambda["arns"]["basichandlerarn"]
+    timestamp = int(execute_lambda["execution_time"].timestamp() * 1000)
+    cw_client = boto3.client("logs")
 
+    # WHEN
+    filtered_logs = helpers.get_logs(
+        lambda_function_name=lambda_arn.split(":")[-1], start_time=timestamp, log_client=cw_client
+    )
+
+    # THEN
     assert any(log.message == {} for log in filtered_logs)
 
 
 @pytest.mark.e2e
-def test_no_context_lambda_contextual_data_not_logged(deploy_lambdas):
-    filtered_logs = fetch_logs(lambda_arn=deploy_lambdas["nocontexthandlerarn"])
+def test_no_context_lambda_contextual_data_not_logged(execute_lambda):
+
+    # GIVEN
+    lambda_arn = execute_lambda["arns"]["nocontexthandlerarn"]
+    timestamp = int(execute_lambda["execution_time"].timestamp() * 1000)
+    cw_client = boto3.client("logs")
+
+    # WHEN
+    filtered_logs = helpers.get_logs(
+        lambda_function_name=lambda_arn.split(":")[-1], start_time=timestamp, log_client=cw_client
+    )
 
+    # THEN
     assert not any(
         (
             log.xray_trace_id
@@ -115,9 +145,19 @@ def test_no_context_lambda_contextual_data_not_logged(deploy_lambdas):
 
 
 @pytest.mark.e2e
-def test_no_context_lambda_event_not_logged(deploy_lambdas):
-    filtered_logs = fetch_logs(lambda_arn=deploy_lambdas["nocontexthandlerarn"])
+def test_no_context_lambda_event_not_logged(execute_lambda):
+
+    # GIVEN
+    lambda_arn = execute_lambda["arns"]["nocontexthandlerarn"]
+    timestamp = int(execute_lambda["execution_time"].timestamp() * 1000)
+    cw_client = boto3.client("logs")
+
+    # WHEN
+    filtered_logs = helpers.get_logs(
+        lambda_function_name=lambda_arn.split(":")[-1], start_time=timestamp, log_client=cw_client
+    )
 
+    # THEN
     assert not any(log.message == {} for log in filtered_logs)