feat(parser): add support for SQS-wrapped S3 event notifications

aws_lambda_powertools/utilities/parser/models/__init__.py

@@ -50,6 +50,10 @@
     S3Model,
     S3RecordModel,
 )
+from .s3_event_notification import (
+    S3SqsEventNotificationModel,
+    S3SqsEventNotificationRecordModel,
+)
 from .s3_object_event import (
     S3ObjectConfiguration,
     S3ObjectContext,
@@ -130,6 +134,8 @@
     "SqsRecordModel",
     "SqsMsgAttributeModel",
     "SqsAttributesModel",
+    "S3SqsEventNotificationModel",
+    "S3SqsEventNotificationRecordModel",
     "APIGatewayProxyEventModel",
     "APIGatewayEventRequestContext",
     "APIGatewayEventAuthorizer",

aws_lambda_powertools/utilities/parser/models/s3_event_notification.py

@@ -0,0 +1,14 @@
+from typing import List
+
+from pydantic import Json
+
+from aws_lambda_powertools.utilities.parser.models.s3 import S3Model
+from aws_lambda_powertools.utilities.parser.models.sqs import SqsModel, SqsRecordModel
+
+
+class S3SqsEventNotificationRecordModel(SqsRecordModel):
+    body: Json[S3Model]
+
+
+class S3SqsEventNotificationModel(SqsModel):
+    Records: List[S3SqsEventNotificationRecordModel]

aws_lambda_powertools/utilities/parser/models/sqs.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import Dict, List, Optional, Type, Union
+from typing import Dict, List, Optional, Sequence, Type, Union
 
 from pydantic import BaseModel
 
@@ -63,4 +63,4 @@ class SqsRecordModel(BaseModel):
 
 
 class SqsModel(BaseModel):
-    Records: List[SqsRecordModel]
+    Records: Sequence[SqsRecordModel]

docs/utilities/parser.md

@@ -156,25 +156,26 @@ def my_function():
 
 Parser comes with the following built-in models:
 
-| Model name                              | Description                                                                  |
-| --------------------------------------- | ---------------------------------------------------------------------------- |
-| **DynamoDBStreamModel**                 | Lambda Event Source payload for Amazon DynamoDB Streams                      |
-| **EventBridgeModel**                    | Lambda Event Source payload for Amazon EventBridge                           |
-| **SqsModel**                            | Lambda Event Source payload for Amazon SQS                                   |
-| **AlbModel**                            | Lambda Event Source payload for Amazon Application Load Balancer             |
-| **CloudwatchLogsModel**                 | Lambda Event Source payload for Amazon CloudWatch Logs                       |
-| **S3Model**                             | Lambda Event Source payload for Amazon S3                                    |
-| **S3ObjectLambdaEvent**                 | Lambda Event Source payload for Amazon S3 Object Lambda                      |
-| **S3EventNotificationEventBridgeModel** | Lambda Event Source payload for Amazon S3 Event Notification to EventBridge. |
-| **KinesisDataStreamModel**              | Lambda Event Source payload for Amazon Kinesis Data Streams                  |
-| **KinesisFirehoseModel**                | Lambda Event Source payload for Amazon Kinesis Firehose                      |
-| **SesModel**                            | Lambda Event Source payload for Amazon Simple Email Service                  |
-| **SnsModel**                            | Lambda Event Source payload for Amazon Simple Notification Service           |
-| **APIGatewayProxyEventModel**           | Lambda Event Source payload for Amazon API Gateway                           |
-| **APIGatewayProxyEventV2Model**         | Lambda Event Source payload for Amazon API Gateway v2 payload                |
-| **LambdaFunctionUrlModel**              | Lambda Event Source payload for Lambda Function URL payload                  |
-| **KafkaSelfManagedEventModel**          | Lambda Event Source payload for self managed Kafka payload                   |
-| **KafkaMskEventModel**                  | Lambda Event Source payload for AWS MSK payload                              |
+| Model name                              | Description                                                                           |
+| --------------------------------------- | ------------------------------------------------------------------------------------- |
+| **AlbModel**                            | Lambda Event Source payload for Amazon Application Load Balancer                      |
+| **APIGatewayProxyEventModel**           | Lambda Event Source payload for Amazon API Gateway                                    |
+| **APIGatewayProxyEventV2Model**         | Lambda Event Source payload for Amazon API Gateway v2 payload                         |
+| **CloudwatchLogsModel**                 | Lambda Event Source payload for Amazon CloudWatch Logs                                |
+| **DynamoDBStreamModel**                 | Lambda Event Source payload for Amazon DynamoDB Streams                               |
+| **EventBridgeModel**                    | Lambda Event Source payload for Amazon EventBridge                                    |
+| **KafkaMskEventModel**                  | Lambda Event Source payload for AWS MSK payload                                       |
+| **KafkaSelfManagedEventModel**          | Lambda Event Source payload for self managed Kafka payload                            |
+| **KinesisDataStreamModel**              | Lambda Event Source payload for Amazon Kinesis Data Streams                           |
+| **KinesisFirehoseModel**                | Lambda Event Source payload for Amazon Kinesis Firehose                               |
+| **LambdaFunctionUrlModel**              | Lambda Event Source payload for Lambda Function URL payload                           |
+| **S3EventNotificationEventBridgeModel** | Lambda Event Source payload for Amazon S3 Event Notification to EventBridge.          |
+| **S3Model**                             | Lambda Event Source payload for Amazon S3                                             |
+| **S3ObjectLambdaEvent**                 | Lambda Event Source payload for Amazon S3 Object Lambda                               |
+| **S3SqsEventNotificationModel**         | Lambda Event Source payload for S3 event notifications wrapped in SQS event (S3->SQS) |
+| **SesModel**                            | Lambda Event Source payload for Amazon Simple Email Service                           |
+| **SnsModel**                            | Lambda Event Source payload for Amazon Simple Notification Service                    |
+| **SqsModel**                            | Lambda Event Source payload for Amazon SQS                                            |
 
 #### Extending built-in models
 

tests/events/s3SqsEvent.json

@@ -0,0 +1,22 @@
+{
+    "Records":[
+       {
+          "messageId":"ca3e7a89-c358-40e5-8aa0-5da01403c267",
+          "receiptHandle":"AQEBE7XoI7IQRLF7SrpiW9W4BanmOWe8UtVDbv6/CEZYKf/OktSNIb4j689tQfR4k44V/LY20lZ5VpxYt2GTYCsSLKTcBalTJaRX9CKu/hVqy/23sSNiKxnP56D+VLSn+hU275+AP1h4pUL0d9gLdRB2haX8xiM+LcGfis5Jl8BBXtoxKRF60O87O9/NvCmmXLeqkJuexfyEZNyed0fFCRXFXSjbmThG0OIQgcrGI8glBRGPA8htns58VtXFsSaPYNoqP3p5n6+ewKKVLD0lfm+0DlnLKRa+mjvFBaSer9KK1ff+Aq6zJ6HynPwADj+aF70Hwimc2zImYe51SLEF/E2csYlMNZYI/2qXW0m9R7wJ/XDTV4g2+h+BMTxsKnJQ6NQd",
+          "body":"{\"Records\":[{\"eventVersion\":\"2.1\",\"eventSource\":\"aws:s3\",\"awsRegion\":\"us-east-1\",\"eventTime\":\"2023-04-12T20:43:38.021Z\",\"eventName\":\"ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"A1YQ72UWCM96UF\"},\"requestParameters\":{\"sourceIPAddress\":\"93.108.161.96\"},\"responseElements\":{\"x-amz-request-id\":\"YMSSR8BZJ2Y99K6P\",\"x-amz-id-2\":\"6ASrUfj5xpn859fIq+6FXflOex/SKl/rjfiMd7wRzMg/zkHKR22PDpnh7KD3uq//cuOTbdX4DInN5eIs+cR0dY1z2Mc5NDP/\"},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"SNS\",\"bucket\":{\"name\":\"xxx\",\"ownerIdentity\":{\"principalId\":\"A1YQ72UWCM96UF\"},\"arn\":\"arn:aws:s3:::xxx\"},\"object\":{\"key\":\"test.pdf\",\"size\":104681,\"eTag\":\"2e3ad1e983318bbd8e73b080e2997980\",\"versionId\":\"yd3d4HaWOT2zguDLvIQLU6ptDTwKBnQV\",\"sequencer\":\"00643717F9F8B85354\"}}}]}",
+          "attributes":{
+             "ApproximateReceiveCount":"1",
+             "SentTimestamp":"1681332219270",
+             "SenderId":"AIDAJHIPRHEMV73VRJEBU",
+             "ApproximateFirstReceiveTimestamp":"1681332239270"
+          },
+          "messageAttributes":{
+
+          },
+          "md5OfBody":"16f4460f4477d8d693a5abe94fdbbd73",
+          "eventSource":"aws:sqs",
+          "eventSourceARN":"arn:aws:sqs:us-east-1:123456789012:SQS",
+          "awsRegion":"us-east-1"
+       }
+    ]
+ }

tests/functional/parser/test_s3.py

@@ -6,8 +6,7 @@
 from tests.functional.utils import load_event
 
 
-@event_parser(model=S3Model)
-def handle_s3(event: S3Model, _: LambdaContext):
+def assert_s3(event: S3Model):
     records = list(event.Records)
     assert len(records) == 1
     record: S3RecordModel = records[0]
@@ -41,6 +40,11 @@ def handle_s3(event: S3Model, _: LambdaContext):
     assert record.glacierEventData is None
 
 
+@event_parser(model=S3Model)
+def handle_s3(event: S3Model, _: LambdaContext):
+    assert_s3(event)
+
+
 @event_parser(model=S3Model)
 def handle_s3_glacier(event: S3Model, _: LambdaContext):
     records = list(event.Records)

tests/unit/parser/test_s3.py

@@ -1,7 +1,12 @@
+import json
 from datetime import datetime
 
+import pytest
+
+from aws_lambda_powertools.utilities.parser import ValidationError
 from aws_lambda_powertools.utilities.parser.models import (
     S3EventNotificationEventBridgeModel,
+    S3SqsEventNotificationModel,
 )
 from tests.functional.utils import load_event
 
@@ -105,3 +110,36 @@ def test_s3_eventbridge_notification_object_restore_completed_event():
     assert model.detail.requester == raw_event["detail"]["requester"]
     assert model.detail.restore_expiry_time == raw_event["detail"]["restore-expiry-time"]
     assert model.detail.source_storage_class == raw_event["detail"]["source-storage-class"]
+
+
+def test_s3_sqs_event_notification():
+    raw_event = load_event("s3SqsEvent.json")
+    model = S3SqsEventNotificationModel(**raw_event)
+
+    body = json.loads(raw_event["Records"][0]["body"])
+
+    assert model.Records[0].body.Records[0].eventVersion == body["Records"][0]["eventVersion"]
+    assert model.Records[0].body.Records[0].eventSource == body["Records"][0]["eventSource"]
+    assert model.Records[0].body.Records[0].eventTime == datetime.fromisoformat(
+        body["Records"][0]["eventTime"].replace("Z", "+00:00")
+    )
+    assert model.Records[0].body.Records[0].eventName == body["Records"][0]["eventName"]
+
+
+def test_s3_sqs_event_notification_body_invalid_json():
+    raw_event = load_event("s3SqsEvent.json")
+
+    for record in raw_event["Records"]:
+        record["body"] = "invalid body"
+
+    with pytest.raises(ValidationError):
+        S3SqsEventNotificationModel(**raw_event)
+
+
+def test_s3_sqs_event_notification_body_containing_arbitrary_json():
+    raw_event = load_event("s3SqsEvent.json")
+    for record in raw_event["Records"]:
+        record["body"] = {"foo": "bar"}
+
+    with pytest.raises(ValidationError):
+        S3SqsEventNotificationModel(**raw_event)