chore(idempotency): add tests for expires_in_progress

Author: rubenfonseca

aws_lambda_powertools/utilities/idempotency/persistence/base.py

@@ -365,7 +365,7 @@ def save_inprogress(self, data: Dict[str, Any], remaining_time_in_millis: Option
 
                 data_record.in_progress_expiry_timestamp = timestamp
             else:
-                logger.debug("Expires in progress is enabled but we couldn't determine the remaining time left")
+                warnings.warn("Expires in progress is enabled but we couldn't determine the remaining time left")
 
         logger.debug(f"Saving in progress record for idempotency key: {data_record.idempotency_key}")
 

aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py

@@ -180,7 +180,7 @@ def _put_record(self, data_record: DataRecord) -> None:
                     "#in_progress_expiry": self.in_progress_expiry_attr,
                     "#status": self.status_attr,
                 },
-                ExpressionAttributeValues={":now": int(now.timestamp()), ":status": STATUS_CONSTANTS["INPROGRESS"]},
+                ExpressionAttributeValues={":now": int(now.timestamp()), ":inprogress": STATUS_CONSTANTS["INPROGRESS"]},
             )
         except self.table.meta.client.exceptions.ConditionalCheckFailedException:
             logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}")

tests/functional/idempotency/conftest.py

@@ -136,7 +136,7 @@ def expected_params_put_item(hashed_idempotency_key):
             "#in_progress_expiry": "in_progress_expiration",
             "#status": "status",
         },
-        "ExpressionAttributeValues": {":now": stub.ANY, ":status": "INPROGRESS"},
+        "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"},
         "Item": {"expiration": stub.ANY, "id": hashed_idempotency_key, "status": "INPROGRESS"},
         "TableName": "TEST_TABLE",
     }
@@ -156,7 +156,7 @@ def expected_params_put_item_with_validation(hashed_idempotency_key, hashed_vali
             "#now": stub.ANY,
             "#status": "status",
         },
-        "ExpressionAttributeValues": {":now": stub.ANY, ":status": "INPROGRESS"},
+        "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"},
         "Item": {
             "expiration": stub.ANY,
             "id": hashed_idempotency_key,

tests/functional/idempotency/test_idempotency.py

@@ -1,5 +1,7 @@
 import copy
+import datetime
 import sys
+import warnings
 from hashlib import md5
 from unittest.mock import MagicMock
 
@@ -10,7 +12,7 @@
 
 from aws_lambda_powertools.utilities.data_classes import APIGatewayProxyEventV2, event_source
 from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig
-from aws_lambda_powertools.utilities.idempotency.base import _prepare_data
+from aws_lambda_powertools.utilities.idempotency.base import MAX_RETRIES, _prepare_data
 from aws_lambda_powertools.utilities.idempotency.exceptions import (
     IdempotencyAlreadyInProgressError,
     IdempotencyInconsistentStateError,
@@ -386,7 +388,7 @@ def test_idempotent_lambda_expired(
     lambda_context,
 ):
     """
-    Test idempotent decorator when lambda is called with an event it succesfully handled already, but outside of the
+    Test idempotent decorator when lambda is called with an event it successfully handled already, but outside of the
     expiry window
     """
 
@@ -529,7 +531,7 @@ def test_idempotent_lambda_expired_during_request(
     lambda_context,
 ):
     """
-    Test idempotent decorator when lambda is called with an event it succesfully handled already. Persistence store
+    Test idempotent decorator when lambda is called with an event it successfully handled already. Persistence store
     returns inconsistent/rapidly changing result between put_item and get_item calls.
     """
 
@@ -804,6 +806,156 @@ def lambda_handler(event, context):
     stubber.deactivate()
 
 
+@pytest.mark.parametrize(
+    "idempotency_config",
+    [
+        {"use_local_cache": False, "expires_in_progress": True},
+        {"use_local_cache": True, "expires_in_progress": True},
+    ],
+    indirect=True,
+)
+def test_idempotent_lambda_expires_in_progress_before_expire(
+    idempotency_config: IdempotencyConfig,
+    persistence_store: DynamoDBPersistenceLayer,
+    lambda_apigw_event,
+    timestamp_future,
+    lambda_response,
+    hashed_idempotency_key,
+    lambda_context,
+):
+    """
+    Test idempotent decorator when expires_in_progress is on and the event is still in progress, before the
+    lambda expiration window.
+    """
+
+    stubber = stub.Stubber(persistence_store.table.meta.client)
+
+    stubber.add_client_error("put_item", "ConditionalCheckFailedException")
+
+    now = datetime.datetime.now()
+    period = datetime.timedelta(seconds=5)
+    timestamp_expires_in_progress = str(int((now + period).timestamp()))
+
+    expected_params_get_item = {
+        "TableName": TABLE_NAME,
+        "Key": {"id": hashed_idempotency_key},
+        "ConsistentRead": True,
+    }
+    ddb_response_get_item = {
+        "Item": {
+            "id": {"S": hashed_idempotency_key},
+            "expiration": {"N": timestamp_future},
+            "in_progress_expiration": {"N": timestamp_expires_in_progress},
+            "data": {"S": '{"message": "test", "statusCode": 200'},
+            "status": {"S": "INPROGRESS"},
+        }
+    }
+    stubber.add_response("get_item", ddb_response_get_item, expected_params_get_item)
+
+    stubber.activate()
+
+    @idempotent(config=idempotency_config, persistence_store=persistence_store)
+    def lambda_handler(event, context):
+        return lambda_response
+
+    with pytest.raises(IdempotencyAlreadyInProgressError):
+        lambda_handler(lambda_apigw_event, lambda_context)
+
+    stubber.assert_no_pending_responses()
+    stubber.deactivate()
+
+
+@pytest.mark.parametrize(
+    "idempotency_config",
+    [
+        {"use_local_cache": False, "expires_in_progress": True},
+        {"use_local_cache": True, "expires_in_progress": True},
+    ],
+    indirect=True,
+)
+def test_idempotent_lambda_expires_in_progress_after_expire(
+    idempotency_config: IdempotencyConfig,
+    persistence_store: DynamoDBPersistenceLayer,
+    lambda_apigw_event,
+    timestamp_future,
+    lambda_response,
+    hashed_idempotency_key,
+    lambda_context,
+):
+    stubber = stub.Stubber(persistence_store.table.meta.client)
+
+    for _ in range(MAX_RETRIES + 1):
+        stubber.add_client_error("put_item", "ConditionalCheckFailedException")
+
+        one_second_ago = datetime.datetime.now() - datetime.timedelta(seconds=1)
+        expected_params_get_item = {
+            "TableName": TABLE_NAME,
+            "Key": {"id": hashed_idempotency_key},
+            "ConsistentRead": True,
+        }
+        ddb_response_get_item = {
+            "Item": {
+                "id": {"S": hashed_idempotency_key},
+                "expiration": {"N": timestamp_future},
+                "in_progress_expiration": {"N": str(int(one_second_ago.timestamp()))},
+                "data": {"S": '{"message": "test", "statusCode": 200'},
+                "status": {"S": "INPROGRESS"},
+            }
+        }
+        stubber.add_response("get_item", ddb_response_get_item, expected_params_get_item)
+
+    stubber.activate()
+
+    @idempotent(config=idempotency_config, persistence_store=persistence_store)
+    def lambda_handler(event, context):
+        return lambda_response
+
+    with pytest.raises(IdempotencyInconsistentStateError):
+        lambda_handler(lambda_apigw_event, lambda_context)
+
+    stubber.assert_no_pending_responses()
+    stubber.deactivate()
+
+
+@pytest.mark.parametrize(
+    "idempotency_config",
+    [
+        {"use_local_cache": False, "expires_in_progress": True},
+        {"use_local_cache": True, "expires_in_progress": True},
+    ],
+    indirect=True,
+)
+def test_idempotent_lambda_expires_in_progress_unavailable_remaining_time(
+    idempotency_config: IdempotencyConfig,
+    persistence_store: DynamoDBPersistenceLayer,
+    lambda_apigw_event,
+    lambda_response,
+    lambda_context,
+    expected_params_put_item,
+    expected_params_update_item,
+):
+    stubber = stub.Stubber(persistence_store.table.meta.client)
+
+    ddb_response = {}
+    stubber.add_response("put_item", ddb_response, expected_params_put_item)
+    stubber.add_response("update_item", ddb_response, expected_params_update_item)
+
+    stubber.activate()
+
+    @idempotent(config=idempotency_config, persistence_store=persistence_store)
+    def lambda_handler(event, context):
+        return lambda_response
+
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("default")
+        lambda_handler(lambda_apigw_event, lambda_context)
+        assert len(w) == 1
+        assert str(w[-1].message) == "Expires in progress is enabled but we couldn't determine the remaining time left"
+
+    stubber.assert_no_pending_responses()
+    stubber.deactivate()
+
+
 def test_data_record_invalid_status_value():
     data_record = DataRecord("key", status="UNSUPPORTED_STATUS")
     with pytest.raises(IdempotencyInvalidStatusError) as e:

tests/functional/idempotency/utils.py

@@ -27,7 +27,7 @@ def build_idempotency_put_item_stub(
             "#in_progress_expiry": "in_progress_expiration",
             "#status": "status",
         },
-        "ExpressionAttributeValues": {":now": stub.ANY, ":status": "INPROGRESS"},
+        "ExpressionAttributeValues": {":now": stub.ANY, ":inprogress": "INPROGRESS"},
         "Item": {"expiration": stub.ANY, "id": idempotency_key_hash, "status": "INPROGRESS"},
         "TableName": "TEST_TABLE",
     }