From 09f1e9e281111146446d8def08f8b0cdeb615498 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Thu, 20 Jun 2024 11:38:40 +0100 Subject: [PATCH] Adding Idempotency test --- noxfile.py | 35 +++ .../functional/idempotency/_boto3/__init__.py | 0 .../idempotency/{ => _boto3}/conftest.py | 0 .../{ => _boto3}/test_idempotency.py | 179 +------------- .../idempotency/_pydantic/__init__.py | 0 .../test_idempotency_with_pydantic.py | 221 ++++++++++++++++++ .../functional/idempotency/_redis/__init__.py | 0 .../test_redis_layer.py | 0 tests/functional/idempotency/utils.py | 4 +- 9 files changed, 259 insertions(+), 180 deletions(-) create mode 100644 tests/functional/idempotency/_boto3/__init__.py rename tests/functional/idempotency/{ => _boto3}/conftest.py (100%) rename tests/functional/idempotency/{ => _boto3}/test_idempotency.py (91%) create mode 100644 tests/functional/idempotency/_pydantic/__init__.py create mode 100644 tests/functional/idempotency/_pydantic/test_idempotency_with_pydantic.py create mode 100644 tests/functional/idempotency/_redis/__init__.py rename tests/functional/idempotency/{persistence => _redis}/test_redis_layer.py (100%) diff --git a/noxfile.py b/noxfile.py index a379adb9274..6f075bfaffd 100644 --- a/noxfile.py +++ b/noxfile.py @@ -110,6 +110,7 @@ def test_with_boto3_sdk_as_required_package(session: nox.Session): f"{PREFIX_TESTS_FUNCTIONAL}/feature_flags/_boto3/", f"{PREFIX_TESTS_UNIT}/data_classes/_boto3/", f"{PREFIX_TESTS_FUNCTIONAL}/streaming/_boto3/", + f"{PREFIX_TESTS_FUNCTIONAL}/idempotency/_boto3/", ], extras="aws-sdk", ) @@ -158,3 +159,37 @@ def test_with_pydantic_required_package(session: nox.Session, pydantic: str): f"{PREFIX_TESTS_UNIT}/parser/_pydantic/", ], ) + + +@nox.session() +@nox.parametrize("pydantic", ["1.10", "2.0"]) +def test_with_boto3_and_pydantic_required_package(session: nox.Session, pydantic: str): + """Tests that only depends for Boto3 + Pydantic library v1 and v2""" + # Idempotency with custom serializer + + session.install(f"pydantic>={pydantic}") + + build_and_run_test( + session, + folders=[ + f"{PREFIX_TESTS_FUNCTIONAL}/idempotency/_pydantic/", + ], + extras="aws-sdk", + ) + + +@nox.session() +def test_with_redis_and_boto3_sdk_as_required_package(session: nox.Session): + """Tests that depends on Redis library""" + # Idempotency - Redis backend + + # Our Redis tests requires multiprocess library to simulate Race Condition + session.run("pip", "install", "multiprocess") + + build_and_run_test( + session, + folders=[ + f"{PREFIX_TESTS_FUNCTIONAL}/idempotency/_redis/", + ], + extras="redis,aws-sdk", + ) diff --git a/tests/functional/idempotency/_boto3/__init__.py b/tests/functional/idempotency/_boto3/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/functional/idempotency/conftest.py b/tests/functional/idempotency/_boto3/conftest.py similarity index 100% rename from tests/functional/idempotency/conftest.py rename to tests/functional/idempotency/_boto3/conftest.py diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/_boto3/test_idempotency.py similarity index 91% rename from tests/functional/idempotency/test_idempotency.py rename to tests/functional/idempotency/_boto3/test_idempotency.py index d056bd4e95a..c0d1104faf3 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/_boto3/test_idempotency.py @@ -8,7 +8,6 @@ import pytest from botocore import stub from botocore.config import Config -from pydantic import BaseModel from pytest import FixtureRequest from pytest_mock import MockerFixture @@ -32,7 +31,6 @@ IdempotencyInconsistentStateError, IdempotencyInvalidStatusError, IdempotencyKeyError, - IdempotencyModelTypeError, IdempotencyNoSerializationModelError, IdempotencyPersistenceLayerError, IdempotencyValidationError, @@ -47,9 +45,6 @@ from aws_lambda_powertools.utilities.idempotency.serialization.dataclass import ( DataclassSerializer, ) -from aws_lambda_powertools.utilities.idempotency.serialization.pydantic import ( - PydanticSerializer, -) from aws_lambda_powertools.utilities.validation import envelopes, validator from aws_lambda_powertools.warnings import PowertoolsUserWarning from tests.functional.idempotency.utils import ( @@ -61,7 +56,7 @@ from tests.functional.utils import json_serialize, load_event TABLE_NAME = "TEST_TABLE" -TESTS_MODULE_PREFIX = "test-func.tests.functional.idempotency.test_idempotency" +TESTS_MODULE_PREFIX = "test-func.tests.functional.idempotency._boto3.test_idempotency" def get_dataclasses_lib(): @@ -1315,106 +1310,6 @@ def record_handler(record): assert from_dict_called is False, "in case response is None, from_dict should not be called" -@pytest.mark.parametrize("output_serializer_type", ["explicit", "deduced"]) -def test_idempotent_function_serialization_pydantic(output_serializer_type: str): - # GIVEN - config = IdempotencyConfig(use_local_cache=True) - mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} - idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 - persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) - - class PaymentInput(BaseModel): - customer_id: str - transaction_id: str - - class PaymentOutput(BaseModel): - customer_id: str - transaction_id: str - - if output_serializer_type == "explicit": - output_serializer = PydanticSerializer( - model=PaymentOutput, - ) - else: - output_serializer = PydanticSerializer - - @idempotent_function( - data_keyword_argument="payment", - persistence_store=persistence_layer, - config=config, - output_serializer=output_serializer, - ) - def collect_payment(payment: PaymentInput) -> PaymentOutput: - return PaymentOutput(**payment.dict()) - - # WHEN - payment = PaymentInput(**mock_event) - first_call: PaymentOutput = collect_payment(payment=payment) - assert first_call.customer_id == payment.customer_id - assert first_call.transaction_id == payment.transaction_id - assert isinstance(first_call, PaymentOutput) - second_call: PaymentOutput = collect_payment(payment=payment) - assert isinstance(second_call, PaymentOutput) - assert second_call.customer_id == payment.customer_id - assert second_call.transaction_id == payment.transaction_id - - -def test_idempotent_function_serialization_pydantic_failure_no_return_type(): - # GIVEN - config = IdempotencyConfig(use_local_cache=True) - mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} - idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 - persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) - - class PaymentInput(BaseModel): - customer_id: str - transaction_id: str - - class PaymentOutput(BaseModel): - customer_id: str - transaction_id: str - - idempotent_function_decorator = idempotent_function( - data_keyword_argument="payment", - persistence_store=persistence_layer, - config=config, - output_serializer=PydanticSerializer, - ) - with pytest.raises(IdempotencyNoSerializationModelError, match="No serialization model was supplied"): - - @idempotent_function_decorator - def collect_payment(payment: PaymentInput): - return PaymentOutput(**payment.dict()) - - -def test_idempotent_function_serialization_pydantic_failure_bad_type(): - # GIVEN - config = IdempotencyConfig(use_local_cache=True) - mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} - idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 - persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) - - class PaymentInput(BaseModel): - customer_id: str - transaction_id: str - - class PaymentOutput(BaseModel): - customer_id: str - transaction_id: str - - idempotent_function_decorator = idempotent_function( - data_keyword_argument="payment", - persistence_store=persistence_layer, - config=config, - output_serializer=PydanticSerializer, - ) - with pytest.raises(IdempotencyModelTypeError, match="Model type is not inherited from pydantic BaseModel"): - - @idempotent_function_decorator - def collect_payment(payment: PaymentInput) -> dict: - return PaymentOutput(**payment.dict()) - - @pytest.mark.parametrize("output_serializer_type", ["explicit", "deduced"]) def test_idempotent_function_serialization_dataclass(output_serializer_type: str): # GIVEN @@ -1493,37 +1388,6 @@ def collect_payment(payment: PaymentInput): return PaymentOutput(**payment.dict()) -def test_idempotent_function_serialization_dataclass_failure_bad_type(): - # GIVEN - dataclasses = get_dataclasses_lib() - config = IdempotencyConfig(use_local_cache=True) - mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} - idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 - persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) - - @dataclasses.dataclass - class PaymentInput: - customer_id: str - transaction_id: str - - @dataclasses.dataclass - class PaymentOutput: - customer_id: str - transaction_id: str - - idempotent_function_decorator = idempotent_function( - data_keyword_argument="payment", - persistence_store=persistence_layer, - config=config, - output_serializer=PydanticSerializer, - ) - with pytest.raises(IdempotencyModelTypeError, match="Model type is not inherited from pydantic BaseModel"): - - @idempotent_function_decorator - def collect_payment(payment: PaymentInput) -> dict: - return PaymentOutput(**payment.dict()) - - def test_idempotent_function_arbitrary_args_kwargs(): # Scenario to validate we can use idempotent_function with a function # with an arbitrary number of args and kwargs @@ -1804,24 +1668,6 @@ class Foo: assert as_dict == expected_result -def test_idempotent_function_pydantic(): - # Scenario _prepare_data should convert a pydantic to a dict - class Foo(BaseModel): - name: str - - expected_result = {"name": "Bar"} - data = Foo(name="Bar") - as_dict = _prepare_data(data) - assert as_dict == data.dict() - assert as_dict == expected_result - - -@pytest.mark.parametrize("data", [None, "foo", ["foo"], 1, True, {}]) -def test_idempotent_function_other(data): - # All other data types should be left as is - assert _prepare_data(data) == data - - def test_idempotent_function_dataclass_with_jmespath(): # GIVEN dataclasses = get_dataclasses_lib() @@ -1847,29 +1693,6 @@ def collect_payment(payment: Payment): assert result == payment.transaction_id -def test_idempotent_function_pydantic_with_jmespath(): - # GIVEN - config = IdempotencyConfig(event_key_jmespath="transaction_id", use_local_cache=True) - mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} - idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_pydantic_with_jmespath..collect_payment#{hash_idempotency_key(mock_event['transaction_id'])}" # noqa E501 - persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) - - class Payment(BaseModel): - customer_id: str - transaction_id: str - - @idempotent_function(data_keyword_argument="payment", persistence_store=persistence_layer, config=config) - def collect_payment(payment: Payment): - return payment.transaction_id - - # WHEN - payment = Payment(**mock_event) - result = collect_payment(payment=payment) - - # THEN idempotency key assertion happens at MockPersistenceLayer - assert result == payment.transaction_id - - @pytest.mark.parametrize("idempotency_config", [{"use_local_cache": False}], indirect=True) def test_idempotent_lambda_compound_already_completed( idempotency_config: IdempotencyConfig, diff --git a/tests/functional/idempotency/_pydantic/__init__.py b/tests/functional/idempotency/_pydantic/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/functional/idempotency/_pydantic/test_idempotency_with_pydantic.py b/tests/functional/idempotency/_pydantic/test_idempotency_with_pydantic.py new file mode 100644 index 00000000000..e5e488e6f49 --- /dev/null +++ b/tests/functional/idempotency/_pydantic/test_idempotency_with_pydantic.py @@ -0,0 +1,221 @@ +import pytest +from pydantic import BaseModel + +from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, + idempotent_function, +) +from aws_lambda_powertools.utilities.idempotency.base import ( + _prepare_data, +) +from aws_lambda_powertools.utilities.idempotency.exceptions import ( + IdempotencyModelTypeError, + IdempotencyNoSerializationModelError, +) +from aws_lambda_powertools.utilities.idempotency.persistence.base import ( + BasePersistenceLayer, + DataRecord, +) +from aws_lambda_powertools.utilities.idempotency.serialization.pydantic import ( + PydanticSerializer, +) +from tests.functional.idempotency.utils import ( + hash_idempotency_key, +) + +TESTS_MODULE_PREFIX = "test-func.tests.functional.idempotency._pydantic.test_idempotency_with_pydantic" + + +def get_dataclasses_lib(): + """Python 3.6 doesn't support dataclasses natively""" + import dataclasses + + return dataclasses + + +class MockPersistenceLayer(BasePersistenceLayer): + def __init__(self, expected_idempotency_key: str): + self.expected_idempotency_key = expected_idempotency_key + super(MockPersistenceLayer, self).__init__() + + def _put_record(self, data_record: DataRecord) -> None: + assert data_record.idempotency_key == self.expected_idempotency_key + + def _update_record(self, data_record: DataRecord) -> None: + assert data_record.idempotency_key == self.expected_idempotency_key + + def _get_record(self, idempotency_key) -> DataRecord: ... + + def _delete_record(self, data_record: DataRecord) -> None: ... + + +@pytest.mark.parametrize("output_serializer_type", ["explicit", "deduced"]) +def test_idempotent_function_serialization_pydantic(output_serializer_type: str): + # GIVEN + config = IdempotencyConfig(use_local_cache=True) + mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + class PaymentInput(BaseModel): + customer_id: str + transaction_id: str + + class PaymentOutput(BaseModel): + customer_id: str + transaction_id: str + + if output_serializer_type == "explicit": + output_serializer = PydanticSerializer( + model=PaymentOutput, + ) + else: + output_serializer = PydanticSerializer + + @idempotent_function( + data_keyword_argument="payment", + persistence_store=persistence_layer, + config=config, + output_serializer=output_serializer, + ) + def collect_payment(payment: PaymentInput) -> PaymentOutput: + return PaymentOutput(**payment.dict()) + + # WHEN + payment = PaymentInput(**mock_event) + first_call: PaymentOutput = collect_payment(payment=payment) + assert first_call.customer_id == payment.customer_id + assert first_call.transaction_id == payment.transaction_id + assert isinstance(first_call, PaymentOutput) + second_call: PaymentOutput = collect_payment(payment=payment) + assert isinstance(second_call, PaymentOutput) + assert second_call.customer_id == payment.customer_id + assert second_call.transaction_id == payment.transaction_id + + +def test_idempotent_function_serialization_pydantic_failure_no_return_type(): + # GIVEN + config = IdempotencyConfig(use_local_cache=True) + mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + class PaymentInput(BaseModel): + customer_id: str + transaction_id: str + + class PaymentOutput(BaseModel): + customer_id: str + transaction_id: str + + idempotent_function_decorator = idempotent_function( + data_keyword_argument="payment", + persistence_store=persistence_layer, + config=config, + output_serializer=PydanticSerializer, + ) + with pytest.raises(IdempotencyNoSerializationModelError, match="No serialization model was supplied"): + + @idempotent_function_decorator + def collect_payment(payment: PaymentInput): + return PaymentOutput(**payment.dict()) + + +def test_idempotent_function_serialization_pydantic_failure_bad_type(): + # GIVEN + config = IdempotencyConfig(use_local_cache=True) + mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + class PaymentInput(BaseModel): + customer_id: str + transaction_id: str + + class PaymentOutput(BaseModel): + customer_id: str + transaction_id: str + + idempotent_function_decorator = idempotent_function( + data_keyword_argument="payment", + persistence_store=persistence_layer, + config=config, + output_serializer=PydanticSerializer, + ) + with pytest.raises(IdempotencyModelTypeError, match="Model type is not inherited from pydantic BaseModel"): + + @idempotent_function_decorator + def collect_payment(payment: PaymentInput) -> dict: + return PaymentOutput(**payment.dict()) + + +def test_idempotent_function_serialization_dataclass_failure_bad_type(): + # GIVEN + dataclasses = get_dataclasses_lib() + config = IdempotencyConfig(use_local_cache=True) + mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + @dataclasses.dataclass + class PaymentInput: + customer_id: str + transaction_id: str + + @dataclasses.dataclass + class PaymentOutput: + customer_id: str + transaction_id: str + + idempotent_function_decorator = idempotent_function( + data_keyword_argument="payment", + persistence_store=persistence_layer, + config=config, + output_serializer=PydanticSerializer, + ) + with pytest.raises(IdempotencyModelTypeError, match="Model type is not inherited from pydantic BaseModel"): + + @idempotent_function_decorator + def collect_payment(payment: PaymentInput) -> dict: + return PaymentOutput(**payment.dict()) + + +def test_idempotent_function_pydantic(): + # Scenario _prepare_data should convert a pydantic to a dict + class Foo(BaseModel): + name: str + + expected_result = {"name": "Bar"} + data = Foo(name="Bar") + as_dict = _prepare_data(data) + assert as_dict == data.dict() + assert as_dict == expected_result + + +@pytest.mark.parametrize("data", [None, "foo", ["foo"], 1, True, {}]) +def test_idempotent_function_other(data): + # All other data types should be left as is + assert _prepare_data(data) == data + + +def test_idempotent_function_pydantic_with_jmespath(): + # GIVEN + config = IdempotencyConfig(event_key_jmespath="transaction_id", use_local_cache=True) + mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_pydantic_with_jmespath..collect_payment#{hash_idempotency_key(mock_event['transaction_id'])}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + class Payment(BaseModel): + customer_id: str + transaction_id: str + + @idempotent_function(data_keyword_argument="payment", persistence_store=persistence_layer, config=config) + def collect_payment(payment: Payment): + return payment.transaction_id + + # WHEN + payment = Payment(**mock_event) + result = collect_payment(payment=payment) + + # THEN idempotency key assertion happens at MockPersistenceLayer + assert result == payment.transaction_id diff --git a/tests/functional/idempotency/_redis/__init__.py b/tests/functional/idempotency/_redis/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/functional/idempotency/persistence/test_redis_layer.py b/tests/functional/idempotency/_redis/test_redis_layer.py similarity index 100% rename from tests/functional/idempotency/persistence/test_redis_layer.py rename to tests/functional/idempotency/_redis/test_redis_layer.py diff --git a/tests/functional/idempotency/utils.py b/tests/functional/idempotency/utils.py index bba8f320438..1bf14393990 100644 --- a/tests/functional/idempotency/utils.py +++ b/tests/functional/idempotency/utils.py @@ -19,7 +19,7 @@ def build_idempotency_put_item_stub( data: Dict, function_name: str = "test-func", function_qualified_name: str = "test_idempotent_lambda_first_execution_event_mutation.", - module_name: str = "tests.functional.idempotency.test_idempotency", + module_name: str = "tests.functional.idempotency._boto3.test_idempotency", handler_name: str = "lambda_handler", ) -> Dict: idempotency_key_hash = ( @@ -57,7 +57,7 @@ def build_idempotency_update_item_stub( handler_response: Dict, function_name: str = "test-func", function_qualified_name: str = "test_idempotent_lambda_first_execution_event_mutation.", - module_name: str = "tests.functional.idempotency.test_idempotency", + module_name: str = "tests.functional.idempotency._boto3.test_idempotency", handler_name: str = "lambda_handler", ) -> Dict: idempotency_key_hash = (