From 2d3b6ab297c7c63419a26521dbc175350c51af58 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 14 May 2025 11:13:12 -0700 Subject: [PATCH 1/9] sync --- .../aws_dbesdk_dynamodb/encrypted/table.py | 310 ++++++++++++++++++ .../python/test/integ/encrypted/README.md | 10 + .../python/test/integ/encrypted/test_table.py | 196 +++++++++++ .../src/basic_put_get_example/__init__.py | 3 + .../with_encrypted_table.py | 151 +++++++++ .../test/basic_put_get_example/__init__.py | 3 + .../test_with_encrypted_table.py | 15 + .../extern/CreateInterceptedDDBTable.py | 226 +++++++++++++ TestVectors/runtimes/python/test/__init__.py | 2 + .../runtimes/python/test/table/__init__.py | 0 .../python/test/table/test_dafny_wrapper.py | 59 ++++ 11 files changed, 975 insertions(+) create mode 100644 DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py create mode 100644 DynamoDbEncryption/runtimes/python/test/integ/encrypted/README.md create mode 100644 DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py create mode 100644 Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/__init__.py create mode 100644 Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/with_encrypted_table.py create mode 100644 Examples/runtimes/python/DynamoDBEncryption/test/basic_put_get_example/__init__.py create mode 100644 Examples/runtimes/python/DynamoDBEncryption/test/basic_put_get_example/test_with_encrypted_table.py create mode 100644 TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBTable.py create mode 100644 TestVectors/runtimes/python/test/__init__.py create mode 100644 TestVectors/runtimes/python/test/table/__init__.py create mode 100644 TestVectors/runtimes/python/test/table/test_dafny_wrapper.py diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py new file mode 100644 index 000000000..4efbaa310 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py @@ -0,0 +1,310 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""High-level helper class to provide an encrypting wrapper for boto3 DynamoDB tables.""" +from collections.abc import Callable +from typing import Any + +from boto3.dynamodb.table import BatchWriter +from boto3.resources.base import ServiceResource + +from aws_dbesdk_dynamodb.encrypted.boto3_interface import EncryptedBotoInterface +from aws_dbesdk_dynamodb.encrypted.client import EncryptedClient +from aws_dbesdk_dynamodb.internal.client_to_resource import ClientShapeToResourceShapeConverter +from aws_dbesdk_dynamodb.internal.resource_to_client import ResourceShapeToClientShapeConverter +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb.models import ( + DynamoDbTablesEncryptionConfig, +) +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb_transforms.client import ( + DynamoDbEncryptionTransforms, +) +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb_transforms.models import ( + GetItemInputTransformInput, + GetItemOutputTransformInput, + PutItemInputTransformInput, + PutItemOutputTransformInput, + QueryInputTransformInput, + QueryOutputTransformInput, + ScanInputTransformInput, + ScanOutputTransformInput, +) + + +class EncryptedTable(EncryptedBotoInterface): + """ + Wrapper for a boto3 DynamoDB table that transparently encrypts/decrypts items. + + This class implements the complete boto3 DynamoDB table API, allowing it to serve as a + drop-in replacement that transparently handles encryption and decryption of items. + + The API matches the standard boto3 DynamoDB table interface: + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table.html + + This class will encrypt/decrypt items for the following operations: + * put_item + * get_item + * query + * scan + + Calling batch_writer() will return a BatchWriter that transparently encrypts batch write requests. + + Any other operations on this class will defer to the underlying boto3 DynamoDB Table's implementation + and will not be encrypted/decrypted. + + Note: The update_item operation is not currently supported. Calling this operation will raise NotImplementedError. + """ + + def __init__( + self, + *, + table: ServiceResource, + encryption_config: DynamoDbTablesEncryptionConfig, + ): + """ + Create an EncryptedTable object. + + Args: + table (ServiceResource): Initialized boto3 DynamoDB table + encryption_config (DynamoDbTablesEncryptionConfig): Initialized DynamoDbTablesEncryptionConfig + + """ + self._table = table + self._encryption_config = encryption_config + self._transformer = DynamoDbEncryptionTransforms(config=encryption_config) + self._client_shape_to_resource_shape_converter = ClientShapeToResourceShapeConverter() + self._resource_shape_to_client_shape_converter = ResourceShapeToClientShapeConverter( + table_name=self._table.table_name + ) + + def put_item(self, **kwargs) -> dict[str, Any]: + """ + Put a single item to the table. Encrypts the item before writing to DynamoDB. + + The parameters and return value match the boto3 DynamoDB table put_item API: + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/put_item.html + + Args: + **kwargs: Keyword arguments to pass to the operation. These match the boto3 put_item API parameters. + The "Item" field will be encrypted locally before being written to DynamoDB. + + Returns: + dict: The response from DynamoDB. This matches the boto3 put_item API response. + + """ + return self._table_operation_logic( + operation_input=kwargs, + input_encryption_transform_method=self._transformer.put_item_input_transform, + input_encryption_transform_shape=PutItemInputTransformInput, + input_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.put_item_request, + input_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.put_item_request, + output_encryption_transform_method=self._transformer.put_item_output_transform, + output_encryption_transform_shape=PutItemOutputTransformInput, + output_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.put_item_response, + output_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.put_item_response, + table_method=self._table.put_item, + ) + + def get_item(self, **kwargs) -> dict[str, Any]: + """ + Get a single item from the table. Decrypts the item after reading from DynamoDB. + + The parameters and return value match the boto3 DynamoDB table get_item API: + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/get_item.html + + Args: + **kwargs: Keyword arguments to pass to the operation. These match the boto3 get_item API parameters. + + Returns: + dict: The response from DynamoDB. This matches the boto3 get_item API response. + The "Item" field will be decrypted locally after being read from DynamoDB. + + """ + return self._table_operation_logic( + operation_input=kwargs, + input_encryption_transform_method=self._transformer.get_item_input_transform, + input_encryption_transform_shape=GetItemInputTransformInput, + input_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.get_item_request, + input_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.get_item_request, + output_encryption_transform_method=self._transformer.get_item_output_transform, + output_encryption_transform_shape=GetItemOutputTransformInput, + output_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.get_item_response, + output_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.get_item_response, + table_method=self._table.get_item, + ) + + def query(self, **kwargs) -> dict[str, Any]: + """ + Query items from the table or index. Decrypts any returned items. + + The parameters and return value match the boto3 DynamoDB table query API: + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/query.html + + Args: + **kwargs: Keyword arguments to pass to the operation. These match the boto3 query API parameters. + + Returns: + dict: The response from DynamoDB. This matches the boto3 query API response. + The "Items" field will be decrypted locally after being read from DynamoDB. + + """ + return self._table_operation_logic( + operation_input=kwargs, + input_encryption_transform_method=self._transformer.query_input_transform, + input_encryption_transform_shape=QueryInputTransformInput, + input_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.query_request, + input_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.query_request, + output_encryption_transform_method=self._transformer.query_output_transform, + output_encryption_transform_shape=QueryOutputTransformInput, + output_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.query_response, + output_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.query_response, + table_method=self._table.query, + ) + + def scan(self, **kwargs) -> dict[str, Any]: + """ + Scan the entire table or index. Decrypts any returned items. + + The parameters and return value match the boto3 DynamoDB table scan API: + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/scan.html + + Args: + **kwargs: Keyword arguments to pass to the operation. These match the boto3 scan API parameters. + + Returns: + dict: The response from DynamoDB. This matches the boto3 scan API response. + The "Items" field will be decrypted locally after being read from DynamoDB. + + """ + return self._table_operation_logic( + operation_input=kwargs, + input_encryption_transform_method=self._transformer.scan_input_transform, + input_encryption_transform_shape=ScanInputTransformInput, + input_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.scan_request, + input_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.scan_request, + output_encryption_transform_method=self._transformer.scan_output_transform, + output_encryption_transform_shape=ScanOutputTransformInput, + output_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.scan_response, + output_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.scan_response, + table_method=self._table.scan, + ) + + def update_item(self, **kwargs): + """ + Not implemented. Raises NotImplementedError. + + Args: + **kwargs: Any arguments passed to this method + + Raises: + NotImplementedError: This operation is not yet implemented + + """ + raise NotImplementedError('"update_item" is not yet implemented') + + def batch_writer(self, overwrite_by_pkeys: list[str] | None = None) -> BatchWriter: + """ + Create a batch writer object that will transparently encrypt requests to DynamoDB. + + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/batch_writer.html + + Args: + overwrite_by_pkeys: De-duplicate request items in buffer if match new request + item on specified primary keys. i.e ``["partition_key1", "sort_key2", "sort_key3"]`` + + Returns: + BatchWriter: A batch writer that will transparently encrypt requests + + """ + encrypted_client = EncryptedClient( + client=self._table.meta.client, + encryption_config=self._encryption_config, + # The boto3 client comes from the underlying table, which is a ServiceResource. + # ServiceResource clients expect standard dictionaries, not DynamoDB JSON. + expect_standard_dictionaries=True, + ) + return BatchWriter(table_name=self._table.name, client=encrypted_client, overwrite_by_pkeys=overwrite_by_pkeys) + + def _table_operation_logic( + self, + *, + operation_input: dict[str, Any], + input_encryption_transform_method: Callable, + input_encryption_transform_shape: Any, + input_resource_to_client_shape_transform_method: Callable, + input_client_to_resource_shape_transform_method: Callable, + output_encryption_transform_method: Callable, + output_encryption_transform_shape: Any, + output_resource_to_client_shape_transform_method: Callable, + output_client_to_resource_shape_transform_method: Any, + table_method: Callable, + ) -> dict[str, Any]: + """ + Interface between user-supplied input, encryption/decryption transformers, and boto3 Tables. + + Args: + operation_input: User-supplied input to the operation + input_encryption_transform_method: The method to transform the input for encryption/decryption + input_encryption_transform_shape: The shape to supply to the input encryption/decryption transform + input_resource_to_client_shape_transform_method: Method to transform resource-formatted input shape + to client-formattted input shape + input_client_to_resource_shape_transform_method: Method to transform client-formatted input shape + to resource-formattted input shape + output_encryption_transform_method: The method to transform the output for encryption/decryption + output_encryption_transform_shape: The shape to supply to the output encryption/decryption transform + output_resource_to_client_shape_transform_method: Method to transform resource-formatted output shape + to client-formattted output shape + output_client_to_resource_shape_transform_method: Method to transform client-formatted output shape + to resource-formattted output shape + table_method: The underlying table method to call + + Returns: + dict: The transformed response from DynamoDB + + """ + # EncryptedTable inputs are formatted as standard dictionaries, but DBESDK transformations expect DynamoDB JSON. + # Convert from standard dictionaries to DynamoDB JSON. + input_transform_input = input_resource_to_client_shape_transform_method(operation_input) + + # Apply DBESDK transformation to the input + input_transform_output = input_encryption_transform_method( + input_encryption_transform_shape(sdk_input=input_transform_input) + ).transformed_input + + # The encryption transformation result is formatted in DynamoDB JSON, + # but the underlying boto3 table expects standard dictionaries. + # Convert from DynamoDB JSON to standard dictionaries. + sdk_input = input_client_to_resource_shape_transform_method(input_transform_output) + + sdk_output = table_method(**sdk_input) + + # Table outputs are formatted as standard dictionaries, but DBESDK transformations expect DynamoDB JSON. + # Convert from standard dictionaries to DynamoDB JSON. + output_transform_input = output_resource_to_client_shape_transform_method(sdk_output) + + # Apply DBESDK transformation to boto3 output + output_transform_output = output_encryption_transform_method( + output_encryption_transform_shape( + original_input=input_transform_input, + sdk_output=output_transform_input, + ) + ).transformed_output + + # EncryptedTable outputs are formatted as standard dictionaries, + # but DBESDK transformations provide DynamoDB JSON. + # Convert from DynamoDB JSON to standard dictionaries. + dbesdk_response = output_client_to_resource_shape_transform_method(output_transform_output) + + # Copy any missing fields from the SDK output to the response (e.g. `ConsumedCapacity`) + dbesdk_response = self._copy_sdk_response_to_dbesdk_response(sdk_output, dbesdk_response) + + return dbesdk_response + + @property + def _boto_client_attr_name(self) -> str: + """ + Name of the attribute containing the underlying boto3 client. + + Returns: + str: '_table' + + """ + return "_table" diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/README.md b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/README.md new file mode 100644 index 000000000..f6a9abf10 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/README.md @@ -0,0 +1,10 @@ +Integration tests for encrypted interfaces. + +These integration tests verify that encrypted boto3 interfaces behave as drop-in replacements for plaintext boto3 interfaces. + +Each test runs with both a plaintext client and an encrypted client, using the same request parameters and expecting the same response. + +This validates that encrypted clients expect the same input shapes as plaintext clients +and encrypted clients return the same output shapes as plaintext clients. + +This guarantees that users can substitute encrypted interfaces without modifying their application logic. diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py new file mode 100644 index 000000000..4ad797c38 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py @@ -0,0 +1,196 @@ +import boto3 +import pytest +from boto3.dynamodb.types import TypeDeserializer, TypeSerializer + +from aws_dbesdk_dynamodb.encrypted.table import EncryptedTable + +from ...constants import ( + INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, + INTEG_TEST_DEFAULT_TABLE_CONFIGS, +) +from ...items import complex_item_dict, simple_item_dict +from ...requests import ( + basic_get_item_request_dict, + basic_put_item_request_dict, + basic_query_request_dict, + basic_scan_request_dict, +) + +serializer = TypeSerializer() +deserializer = TypeDeserializer() + + +def encrypted_table(): + """Create an encrypted table.""" + table = boto3.resource("dynamodb").Table(INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME) + return EncryptedTable( + table=table, + encryption_config=INTEG_TEST_DEFAULT_TABLE_CONFIGS, + ) + + +def plaintext_table(): + """Create a plaintext table.""" + table = boto3.resource("dynamodb").Table(INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME) + return table + + +@pytest.fixture(params=[True, False], ids=["encrypted", "plaintext"]) +def encrypted(request): + return request.param + + +@pytest.fixture +def table(encrypted): + """ + Create a table client. + Use both to test that the same input can be provided to both boto3 and the EncryptedTable. + """ + if encrypted: + return encrypted_table() + else: + return plaintext_table() + + +@pytest.fixture(params=[simple_item_dict, complex_item_dict], ids=["simple_item", "complex_item"]) +def test_item(request): + return request.param + + +def test_GIVEN_item_WHEN_basic_put_and_basic_get_THEN_round_trip_passes(table, test_item): + """Test put_item and get_item operations.""" + # Given: Simple and complex items in appropriate format for client + put_item_request_dict = basic_put_item_request_dict(test_item) + + # When: Putting and getting item + put_response = table.put_item(**put_item_request_dict) + assert put_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + get_item_request_dict = basic_get_item_request_dict(test_item) + get_response = table.get_item(**get_item_request_dict) + # Then: Simple item is encrypted and decrypted correctly + assert get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert get_response["Item"] == put_item_request_dict["Item"] + + +def test_GIVEN_items_WHEN_batch_write_and_get_THEN_round_trip_passes( + table, +): + # Given: Simple and complex items in appropriate format for client + # When: Batch put items + with table.batch_writer() as batch_writer: + # boto3 documentation for batch_writer.put_item() is incorrect; + # the method accepts the item directly, not the item inside an "Item" key. + batch_writer.put_item(simple_item_dict) + batch_writer.put_item(complex_item_dict) + + # When: Get items + get_item_request_dict = basic_get_item_request_dict(simple_item_dict) + get_response = table.get_item(**get_item_request_dict) + # Then: All items are encrypted and decrypted correctly + assert get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert get_response["Item"] == simple_item_dict + + get_item_request_dict = basic_get_item_request_dict(complex_item_dict) + get_response = table.get_item(**get_item_request_dict) + # Then: All items are encrypted and decrypted correctly + assert get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert get_response["Item"] == complex_item_dict + + # When: Batch delete items + with table.batch_writer() as batch_writer: + batch_writer.delete_item( + {"partition_key": simple_item_dict["partition_key"], "sort_key": simple_item_dict["sort_key"]} + ) + batch_writer.delete_item( + {"partition_key": complex_item_dict["partition_key"], "sort_key": complex_item_dict["sort_key"]} + ) + + # When: Get items + get_item_request_dict = basic_get_item_request_dict(simple_item_dict) + get_response = table.get_item(**get_item_request_dict) + # Then: All items are encrypted and decrypted correctly + assert get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert "Item" not in get_response + + get_item_request_dict = basic_get_item_request_dict(complex_item_dict) + get_response = table.get_item(**get_item_request_dict) + # Then: All items are encrypted and decrypted correctly + assert get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert "Item" not in get_response + + +def test_GIVEN_items_in_table_WHEN_query_THEN_items_are_decrypted_correctly(table, test_item): + """Test query and scan operations.""" + # Given: Simple and complex items in appropriate format for client + # When: Putting items into table + put_item_request_dict = basic_put_item_request_dict(test_item) + table.put_item(**put_item_request_dict) + + # When: Querying items by partition key + query_request_dict = basic_query_request_dict(test_item) + query_response = table.query(**query_request_dict) + # Then: Query returns correct items + assert query_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert len(query_response["Items"]) == 1 + assert query_response["Items"][0] == put_item_request_dict["Item"] + + # Scans work, but the test items are not found because + # DDB only returns the first 1MB of data, and the test items + # are not in the first 1MB sometimes. We probably need a new table. + # TODO: Add a new table for these tests, enable tests. + # # When: Scanning with filter that matches only our test items + # scan_response = encrypted_table.scan(**scan_request_dict) + # # Then: Scan returns both test items + # assert scan_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + # assert len(scan_response["Items"]) == 2 + # # Check each test item is found in scan results + # found_items = scan_response["Items"] + # assert all(any(found_item == item for found_item in found_items) for item in items) + + +@pytest.fixture +def scan_request(encrypted, test_item): + if encrypted: + request = basic_scan_request_dict(test_item) + request["FilterExpression"] = request["FilterExpression"] + " AND attribute_exists (#sig)" + request["ExpressionAttributeNames"] = {} + request["ExpressionAttributeNames"]["#sig"] = "amzn-ddb-map-sig" + return request + return basic_scan_request_dict(test_item) + + +def test_GIVEN_valid_put_and_scan_requests_WHEN_put_and_scan_THEN_round_trip_passes(table, test_item, scan_request): + """Test put_item and scan operations.""" + # Given: Simple and complex items in appropriate format for client + put_item_request_dict = basic_put_item_request_dict(test_item) + table.put_item(**put_item_request_dict) + + # When: Scanning items + scan_request_dict = scan_request + scan_response = table.scan(**scan_request_dict) + # Then: Scan returns both test items + assert scan_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + +def test_WHEN_update_item_THEN_raises_not_implemented_error(): + # Given: Encrypted client and update item parameters + # When: Calling update_item + with pytest.raises(NotImplementedError): + encrypted_table().update_item( + TableName=INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, + Key={"partition_key": "test-key", "sort_key": 1}, + UpdateExpression="SET attribute1 = :val", + ExpressionAttributeValues={":val": {"S": "new value"}}, + ) + # Then: NotImplementedError is raised + + +def test_WHEN_call_passthrough_method_THEN_correct_response_is_returned(): + """Test that calling a passthrough method returns the correct response.""" + # Given: Encrypted client + # When: Calling some passthrough method that does not explicitly exist on EncryptedClient, + # but exists on the underlying boto3 client + response = encrypted_table().meta.client.list_backups() + # Then: Correct response is returned, i.e. EncryptedClient forwards the call to the underlying boto3 client + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 diff --git a/Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/__init__.py b/Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/__init__.py new file mode 100644 index 000000000..fa977e22f --- /dev/null +++ b/Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub to allow relative imports of examples from tests.""" diff --git a/Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/with_encrypted_table.py b/Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/with_encrypted_table.py new file mode 100644 index 000000000..d038e0dba --- /dev/null +++ b/Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/with_encrypted_table.py @@ -0,0 +1,151 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +Example for using an EncryptedTable to put and get an encrypted item. + +Running this example requires access to the DDB Table whose name +is provided in the function arguments. +This table must be configured with the following +primary key configuration: +- Partition key is named "partition_key" with type (S) +- Sort key is named "sort_key" with type (N) +""" + +import boto3 +from aws_cryptographic_material_providers.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_material_providers.mpl.config import MaterialProvidersConfig +from aws_cryptographic_material_providers.mpl.models import ( + CreateAwsKmsMrkMultiKeyringInput, + DBEAlgorithmSuiteId, +) +from aws_cryptographic_material_providers.mpl.references import IKeyring +from aws_dbesdk_dynamodb.encrypted.table import EncryptedTable +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb.models import ( + DynamoDbTableEncryptionConfig, + DynamoDbTablesEncryptionConfig, +) +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_structuredencryption.models import ( + CryptoAction, +) + + +def encrypted_table_put_get_example( + kms_key_id: str, + dynamodb_table_name: str, +): + """Use an EncryptedTable to put and get an encrypted item.""" + # 1. Create a Keyring. This Keyring will be responsible for protecting the data keys that protect your data. + # For this example, we will create a AWS KMS Keyring with the AWS KMS Key we want to use. + # We will use the `CreateMrkMultiKeyring` method to create this keyring, + # as it will correctly handle both single region and Multi-Region KMS Keys. + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders(config=MaterialProvidersConfig()) + kms_mrk_multi_keyring_input: CreateAwsKmsMrkMultiKeyringInput = CreateAwsKmsMrkMultiKeyringInput( + generator=kms_key_id, + ) + kms_mrk_multi_keyring: IKeyring = mat_prov.create_aws_kms_mrk_multi_keyring(input=kms_mrk_multi_keyring_input) + + # 2. Configure which attributes are encrypted and/or signed when writing new items. + # For each attribute that may exist on the items we plan to write to our DynamoDbTable, + # we must explicitly configure how they should be treated during item encryption: + # - ENCRYPT_AND_SIGN: The attribute is encrypted and included in the signature + # - SIGN_ONLY: The attribute not encrypted, but is still included in the signature + # - DO_NOTHING: The attribute is not encrypted and not included in the signature + attribute_actions_on_encrypt = { + "partition_key": CryptoAction.SIGN_ONLY, + "sort_key": CryptoAction.SIGN_ONLY, + "attribute1": CryptoAction.ENCRYPT_AND_SIGN, + "attribute2": CryptoAction.SIGN_ONLY, + ":attribute3": CryptoAction.DO_NOTHING, + } + + # 3. Configure which attributes we expect to be included in the signature + # when reading items. There are two options for configuring this: + # + # - (Recommended) Configure `allowedUnsignedAttributesPrefix`: + # When defining your DynamoDb schema and deciding on attribute names, + # choose a distinguishing prefix (such as ":") for all attributes that + # you do not want to include in the signature. + # This has two main benefits: + # - It is easier to reason about the security and authenticity of data within your item + # when all unauthenticated data is easily distinguishable by their attribute name. + # - If you need to add new unauthenticated attributes in the future, + # you can easily make the corresponding update to your `attributeActionsOnEncrypt` + # and immediately start writing to that new attribute, without + # any other configuration update needed. + # Once you configure this field, it is not safe to update it. + # + # - Configure `allowedUnsignedAttributes`: You may also explicitly list + # a set of attributes that should be considered unauthenticated when encountered + # on read. Be careful if you use this configuration. Do not remove an attribute + # name from this configuration, even if you are no longer writing with that attribute, + # as old items may still include this attribute, and our configuration needs to know + # to continue to exclude this attribute from the signature scope. + # If you add new attribute names to this field, you must first deploy the update to this + # field to all readers in your host fleet before deploying the update to start writing + # with that new attribute. + # + # For this example, we have designed our DynamoDb table such that any attribute name with + # the ":" prefix should be considered unauthenticated. + unsignAttrPrefix: str = ":" + + # 4. Create the DynamoDb Encryption configuration for the table we will be writing to. + table_configs = {} + table_config = DynamoDbTableEncryptionConfig( + logical_table_name=dynamodb_table_name, + partition_key_name="partition_key", + sort_key_name="sort_key", + attribute_actions_on_encrypt=attribute_actions_on_encrypt, + keyring=kms_mrk_multi_keyring, + allowed_unsigned_attribute_prefix=unsignAttrPrefix, + # Specifying an algorithm suite is not required, + # but is done here to demonstrate how to do so. + # We suggest using the + # `ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384_SYMSIG_HMAC_SHA384` suite, + # which includes AES-GCM with key derivation, signing, and key commitment. + # This is also the default algorithm suite if one is not specified in this config. + # For more information on supported algorithm suites, see: + # https://docs.aws.amazon.com/database-encryption-sdk/latest/devguide/supported-algorithms.html + algorithm_suite_id=DBEAlgorithmSuiteId.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384_SYMSIG_HMAC_SHA384, + ) + table_configs[dynamodb_table_name] = table_config + tables_config = DynamoDbTablesEncryptionConfig(table_encryption_configs=table_configs) + + # 5. Create the EncryptedClient + encrypted_table = EncryptedTable( + table=boto3.resource("dynamodb").Table(dynamodb_table_name), + encryption_config=tables_config, + ) + + # 6. Put an item into our table using the above client. + # Before the item gets sent to DynamoDb, it will be encrypted + # client-side, according to our configuration. + item_to_encrypt = { + "partition_key": "BasicPutGetExample", + "sort_key": 0, + "attribute1": "encrypt and sign me!", + "attribute2": "sign me!", + ":attribute3": "ignore me!", + } + + put_item_request = { + "TableName": dynamodb_table_name, + "Item": item_to_encrypt, + } + + put_item_response = encrypted_table.put_item(**put_item_request) + + # Demonstrate that PutItem succeeded + assert put_item_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # 7. Get the item back from our table using the same client. + # The client will decrypt the item client-side, and return + # back the original item. + key_to_get = {"partition_key": "BasicPutGetExample", "sort_key": 0} + + get_item_request = {"TableName": dynamodb_table_name, "Key": key_to_get} + + get_item_response = encrypted_table.get_item(**get_item_request) + + # Demonstrate that GetItem succeeded and returned the decrypted item + assert get_item_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert get_item_response["Item"] == item_to_encrypt diff --git a/Examples/runtimes/python/DynamoDBEncryption/test/basic_put_get_example/__init__.py b/Examples/runtimes/python/DynamoDBEncryption/test/basic_put_get_example/__init__.py new file mode 100644 index 000000000..fa977e22f --- /dev/null +++ b/Examples/runtimes/python/DynamoDBEncryption/test/basic_put_get_example/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub to allow relative imports of examples from tests.""" diff --git a/Examples/runtimes/python/DynamoDBEncryption/test/basic_put_get_example/test_with_encrypted_table.py b/Examples/runtimes/python/DynamoDBEncryption/test/basic_put_get_example/test_with_encrypted_table.py new file mode 100644 index 000000000..5a5c665de --- /dev/null +++ b/Examples/runtimes/python/DynamoDBEncryption/test/basic_put_get_example/test_with_encrypted_table.py @@ -0,0 +1,15 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test suite for the EncryptedClient example.""" +import pytest + +from ...src.basic_put_get_example.with_encrypted_table import encrypted_table_put_get_example + +pytestmark = [pytest.mark.examples] + + +def test_encrypted_table_put_get_example(): + """Test function for encrypt and decrypt using the EncryptedClient example.""" + test_kms_key_id = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" + test_dynamodb_table_name = "DynamoDbEncryptionInterceptorTestTable" + encrypted_table_put_get_example(test_kms_key_id, test_dynamodb_table_name) diff --git a/TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBTable.py b/TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBTable.py new file mode 100644 index 000000000..e27df97f5 --- /dev/null +++ b/TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBTable.py @@ -0,0 +1,226 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import boto3 +import aws_dbesdk_dynamodb_test_vectors.internaldafny.generated.CreateInterceptedDDBClient +import aws_cryptography_internal_dynamodb.internaldafny.extern +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb.dafny_to_smithy import aws_cryptography_dbencryptionsdk_dynamodb_DynamoDbTablesEncryptionConfig +from aws_dbesdk_dynamodb.encrypted.table import ( + EncryptedTable, +) +from aws_dbesdk_dynamodb.internal.resource_to_client import ResourceShapeToClientShapeConverter +from aws_dbesdk_dynamodb.internal.client_to_resource import ClientShapeToResourceShapeConverter +from smithy_dafny_standard_library.internaldafny.generated import Wrappers +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb.errors import _smithy_error_to_dafny_error +from aws_dbesdk_dynamodb_test_vectors.waiting_boto3_ddb_client import WaitingLocalDynamoClient +from aws_dbesdk_dynamodb.transform import ( + dict_to_ddb, + ddb_to_dict, +) +from aws_dbesdk_dynamodb.internal import client_to_resource + +from boto3.dynamodb.conditions import Key, Attr, And, Or, Not, Contains +from boto3.dynamodb.types import TypeDeserializer + +# from .....test.resource_formatted_queries import (queries, complex_queries) + +# When querying, DBESDK DDB TestVectors will pass the Table the query as a string. +# The Table could accept this string as-is and process it correctly. +# However, EncryptedTables have extra logic to process boto3 Conditions. +# I want to test this extra logic as much as possible. +# This map converts some known query strings to equivalent Conditions. +# TestVectors will pass the query string (map key) to the Table; +# the Table's internal logic will look up the query string in this map: +# - Entry found: Query with replaced Condition +# - Not found: Query with original string. Table accepts strings. +# This map contains all query strings in the TestVectors' data.json as of commit +# 4f18689f79243c9a5ab0f3a23108671defddeac4 +# If any query strings are added to TestVectors, they COULD be added here, +# but do not need to be added. +known_query_string_to_condition_map = { + # "Basic" queries + "RecNum = :zero": Key("RecNum").eq(":zero"), + "RecNum = :one": Key("RecNum").eq(":one"), + "RecNum = :zero": Attr("RecNum").eq(":zero"), + "RecNum <= :zero": Attr("RecNum").lte(":zero"), + "RecNum > :zero": Attr("RecNum").gt(":zero"), + "RecNum >= :zero": Attr("RecNum").gte(":zero"), + "RecNum <> :zero": Attr("RecNum").ne(":zero"), + "RecNum = :zero": Attr("RecNum").eq(":zero"), + "RecNum = :one": Attr("RecNum").eq(":one"), + "Nine between :zeroD and :three": Attr("Nine").between(":zeroD", ":three"), + "Nine between :nineD and :nine": Attr("Nine").between(":nineD", ":nine"), + "Nine between :nine and :three": Attr("Nine").between(":nine", ":three"), + "Nine between :nine and :nine": Attr("Nine").between(":nine", ":nine"), + "NumberTest = :NumberTest": Attr("NumberTest").eq(":NumberTest"), + "RecNum in (:zero, :one)": Attr("RecNum").is_in([":zero", ":one"]), + "Two = :two": Attr("Two").eq(":two"), + "Two = :two or Three = :three or Four = :four OR Five = :five": Attr("Two").eq(":two") | Attr("Three").eq(":three") | Attr("Four").eq(":four") | Attr("Five").eq(":five"), + "Two = :two and Three = :three and Four = :four and Five = :five": Attr("Two").eq(":two") & Attr("Three").eq(":three") & Attr("Four").eq(":four") & Attr("Five").eq(":five"), + "Two in (:two, :three, :four, :five)": Attr("Two").is_in([":two", ":three", ":four", ":five"]), + "Five in (:two, :three, :four, :five)": Attr("Five").is_in([":two", ":three", ":four", ":five"]), + "Five in (:strset)": Attr("Five").is_in([":strset"]), + "Five in (:strlist)": Attr("Five").is_in([":strlist"]), + "contains(One, :oneA)": Attr("One").contains(":oneA"), + "contains(One, :oneB)": Attr("One").contains(":oneB"), + # Hard-coding returning the input string for these cases. + # These conditions test undocumented behavior in DynamoDB that can't be expressed with boto3 Conditions. + # The undocumented behavior is that `contains`' first parameter can be a value, + # and does not need to be an attribute name. + # DynamoDB documentation names `contains`' first argument as `path`, + # and only ever documents accepting an attribute name for `path`. + # However, testing with an AWS SDK reveals that `path` can be a value; + # i.e. a hardcoded string or an attribute value, + # so this expression is valid. + # But I can't find a way to express this via boto3 Conditions, + # where Contains requires an attribute name. + # For these strings, do not attempt to convert to boto3 conditions, + # and just return the input string. + # The input string is still passed to the table and tested. + "contains(:oneA, One)": "contains(:oneA, One)", + "contains(:oneB, One)": "contains(:oneB, One)", + "contains(:strset, One)": "contains(:strset, One)", + + # "Complex" queries + "Comp1 := :cmp1a": Attr("Comp1").eq(":cmp1a"), + "begins_with(Comp1, :cmp1c)": Attr("Comp1").begins_with(":cmp1c"), + "cmp1c < Comp1": Attr("cmp1c").lt("Comp1"), + "cmp1c = Comp1": Attr("cmp1c").eq("Comp1"), + "begins_with(Comp1, :cmp1d)": Attr("Comp1").begins_with(":cmp1d"), + "contains(Comp1, :cmp1c)": Attr("Comp1").contains(":cmp1c"), + "contains(Comp1, :cmp1d)": Attr("Comp1").contains(":cmp1d"), + "Comp1 = :cmp1b": Attr("Comp1").eq(":cmp1b"), + + # Another query that can't be translated to boto3 Conditions, + # since attribute values aren't attribute names. + # Pass the original string through. + ":cmp1c <= Comp1": ":cmp1c <= Comp1", +} + +class DynamoDBClientWrapperForDynamoDBTable: + """ + DBESDK TestVectors-internal wrapper class. + Converts boto3 DynamoDB client-formatted inputs to Table-formatted inputs, + and converts Table-formatted outputs to boto3 DynamoDB client-formatted outputs. + + TestVectors Dafny code only knows how to interact with DynamoDB clients. + However, Python DDBEC and DBESDK have this EncryptedTable class. + This class interfaces between Dafny TestVectors' DynamoDB client-calling code + and Python DBESDK's EncryptedTable class. + + This class defers to a boto3 client for create_table and delete_table, + which are not supported on boto3 DynamoDB Table tables. + """ + + def __init__(self, table, client): + self._table = table + self._client = client + self._client_shape_to_resource_shape_converter = ClientShapeToResourceShapeConverter() + self._resource_shape_to_client_shape_converter = ResourceShapeToClientShapeConverter(table_name = self._table._table.table_name) + + def put_item(self, **kwargs): + table_input = self._client_shape_to_resource_shape_converter.put_item_request(kwargs) + table_output = self._table.put_item(**table_input) + client_output = self._resource_shape_to_client_shape_converter.put_item_response(table_output) + return client_output + + def get_item(self, **kwargs): + table_input = self._client_shape_to_resource_shape_converter.get_item_request(kwargs) + table_output = self._table.get_item(**table_input) + client_output = self._resource_shape_to_client_shape_converter.get_item_response(table_output) + return client_output + + def batch_write_item(self, **kwargs): + # There isn't a resource shape for this; + table_input = self._client_shape_to_resource_shape_converter.batch_write_item_request(kwargs) + # table_output = self._table.batch_write_item(**table_input) + with self._table.batch_writer() as batch_writer: + for _, items in table_input["RequestItems"].items(): + for item in items: + if "PutRequest" in item: + batch_writer.put_item(item["PutRequest"]["Item"]) + elif "DeleteRequest" in item: + batch_writer.delete_item(item["DeleteRequest"]["Key"]) + else: + raise ValueError(f"Unknown request type: {item}") + # There isn't a shape for the output, but luckily the output can be an empty dict: + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/client/batch_write_item.html + client_output = {} + return client_output + + def batch_get_item(self, **kwargs): + raise NotImplementedError("batch_get_item not supported on table interface; remove tests calling this") + + def scan(self, **kwargs): + table_input = self._client_shape_to_resource_shape_converter.scan_request(kwargs) + # To exhaustively test Tables, + # convert the string-based KeyConditionExpression and FilterExpression + # into the boto3.conditions.Key and boto3.conditions.Attr resource-formatted queries. + if "KeyConditionExpression" in table_input: + if table_input["KeyConditionExpression"] in known_query_string_to_condition_map: + # Turn the query into the resource-formatted query + query = known_query_string_to_condition_map[table_input["KeyConditionExpression"]] + table_input["KeyConditionExpression"] = query + if "FilterExpression" in table_input: + if table_input["FilterExpression"] in known_query_string_to_condition_map: + # Turn the query into the resource-formatted query + table_input["FilterExpression"] = known_query_string_to_condition_map[table_input["FilterExpression"]] + table_output = self._table.scan(**table_input) + client_output = self._resource_shape_to_client_shape_converter.scan_response(table_output) + return client_output + + def transact_get_items(self, **kwargs): + raise NotImplementedError("transact_get_items not supported on table interface; remove tests calling this") + + def transact_write_items(self, **kwargs): + raise NotImplementedError("transact_write_items not supported on table interface; remove tests calling this") + + def query(self, **kwargs): + table_input = self._client_shape_to_resource_shape_converter.query_request(kwargs) + # To exhaustively test Tables, + # convert the string-based KeyConditionExpression and FilterExpression + # into the boto3.conditions.Key and boto3.conditions.Attr resource-formatted queries. + if "KeyConditionExpression" in table_input: + if table_input["KeyConditionExpression"] in known_query_string_to_condition_map: + # Turn the query into the resource-formatted query + query = known_query_string_to_condition_map[table_input["KeyConditionExpression"]] + table_input["KeyConditionExpression"] = query + if "FilterExpression" in table_input: + if table_input["FilterExpression"] in known_query_string_to_condition_map: + # Turn the query into the resource-formatted query + table_input["FilterExpression"] = known_query_string_to_condition_map[table_input["FilterExpression"]] + table_output = self._table.query(**table_input) + client_output = self._resource_shape_to_client_shape_converter.query_response(table_output) + return client_output + + def delete_table(self, **kwargs): + return self._client.delete_table(**kwargs) + + def create_table(self, **kwargs): + return self._client.create_table(**kwargs) + +class default__: + @staticmethod + def CreateVanillaDDBClient(): + try: + return aws_cryptography_internal_dynamodb.internaldafny.extern.Com_Amazonaws_Dynamodb.default__.DynamoDBClient(WaitingLocalDynamoClient()) + except Exception as e: + return Wrappers.Result_Failure(_smithy_error_to_dafny_error(e)) + + @staticmethod + def CreateInterceptedDDBClient(dafny_encryption_config): + try: + native_encryption_config = aws_cryptography_dbencryptionsdk_dynamodb_DynamoDbTablesEncryptionConfig(dafny_encryption_config) + boto3_client = WaitingLocalDynamoClient() + table_config_names = list(native_encryption_config.table_encryption_configs.keys()) + if len(table_config_names) > 1: + # If needed, >1 table could be supported by setting up an EncryptedTablesManager + raise ValueError(">1 table not supported") + # For TestVectors, use local DynamoDB endpoint + table = boto3.resource('dynamodb', endpoint_url="http://localhost:8000").Table(table_config_names[0]) + encrypted_table = EncryptedTable(table = table, encryption_config = native_encryption_config) + wrapped_encrypted_table = DynamoDBClientWrapperForDynamoDBTable(table = encrypted_table, client = boto3_client) + return aws_cryptography_internal_dynamodb.internaldafny.extern.Com_Amazonaws_Dynamodb.default__.DynamoDBClient(wrapped_encrypted_table) + except Exception as e: + return Wrappers.Result_Failure(_smithy_error_to_dafny_error(e)) + +aws_dbesdk_dynamodb_test_vectors.internaldafny.generated.CreateInterceptedDDBClient.default__ = default__ diff --git a/TestVectors/runtimes/python/test/__init__.py b/TestVectors/runtimes/python/test/__init__.py new file mode 100644 index 000000000..f94fd12a2 --- /dev/null +++ b/TestVectors/runtimes/python/test/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/TestVectors/runtimes/python/test/table/__init__.py b/TestVectors/runtimes/python/test/table/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/TestVectors/runtimes/python/test/table/test_dafny_wrapper.py b/TestVectors/runtimes/python/test/table/test_dafny_wrapper.py new file mode 100644 index 000000000..c21c59a58 --- /dev/null +++ b/TestVectors/runtimes/python/test/table/test_dafny_wrapper.py @@ -0,0 +1,59 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +Wrapper file for executing Dafny tests from pytest. +This allows us to import modules required by Dafny-generated tests +before executing Dafny-generated tests. +pytest will find and execute the `test_dafny` method below, +which will execute the `internaldafny_test_executor.py` file in the `dafny` directory. +""" + +import sys +from functools import partial +# Different from standard test_dafny_wrapper due to weird test structure. +test_dir = '/'.join(__file__.split("/")[:-2]) + +sys.path.append(test_dir + "/internaldafny/extern") +sys.path.append(test_dir + "/internaldafny/generated") + +import aws_dbesdk_dynamodb_test_vectors.internaldafny.extern.CreateInterceptedDDBTable +import aws_dbesdk_dynamodb_test_vectors.internaldafny.extern.CreateWrappedDictItemEncryptor + +# Remove invalid tests. +# Supported operations on Tables that are also supported by DBESDK are: +# - put_item +# - get_item +# - query +# - scan +# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/index.html#DynamoDB.Table +# +# Unsupported operations on Tables are that are supported by DBESDK are: +# - transact_get_items +# - transact_write_items +# - batch_get_item +# Remove any tests that call unsupported operations by overriding the test method to do nothing. +# If more tests that call these operations are added, remove them below. +# If the list below becomes unmaintainable, or if other languages add clients with unsupported operations, +# refactor the Dafny code to conditionally call tests based on whether the client supports the operation under test. + +def EmptyTest(*args, **kwargs): + print(f"Skipping test {kwargs['test_name']} because {kwargs['reason']}") + +aws_dbesdk_dynamodb_test_vectors.internaldafny.generated.DdbEncryptionTestVectors.TestVectorConfig.BasicIoTestTransactGetItems = partial( + EmptyTest, + test_name="BasicIoTestTransactGetItems", + reason="DDB tables do not support transact_get_items" +) +aws_dbesdk_dynamodb_test_vectors.internaldafny.generated.DdbEncryptionTestVectors.TestVectorConfig.BasicIoTestTransactWriteItems = partial( + EmptyTest, + test_name="BasicIoTestTransactWriteItems", + reason="DDB tables do not support transact_write_items" +) +aws_dbesdk_dynamodb_test_vectors.internaldafny.generated.DdbEncryptionTestVectors.TestVectorConfig.BasicIoTestBatchGetItems = partial( + EmptyTest, + test_name="BasicIoTestBatchGetItems", + reason="DDB tables do not support batch_get_item" +) + +def test_dafny(): + from ..internaldafny.generated import __main__ \ No newline at end of file From c85fcc05a2ab505b4f16f06a4459ab0fb8d15d96 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 14 May 2025 12:53:56 -0700 Subject: [PATCH 2/9] sync --- .../aws_dbesdk_dynamodb/encrypted/table.py | 58 +++++++++++-------- .../python/test/integ/encrypted/test_table.py | 23 ++++---- .../extern/CreateInterceptedDDBTable.py | 28 +++------ 3 files changed, 52 insertions(+), 57 deletions(-) diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py index 4efbaa310..d4a88e454 100644 --- a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py @@ -37,20 +37,21 @@ class EncryptedTable(EncryptedBotoInterface): drop-in replacement that transparently handles encryption and decryption of items. The API matches the standard boto3 DynamoDB table interface: + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table.html This class will encrypt/decrypt items for the following operations: - * put_item - * get_item - * query - * scan + * ``put_item`` + * ``get_item`` + * ``query`` + * ``scan`` + + The ``update_item`` operation is not currently supported. Calling this operation will raise ``NotImplementedError``. - Calling batch_writer() will return a BatchWriter that transparently encrypts batch write requests. + Calling ``batch_writer()`` will return a ``BatchWriter`` that transparently encrypts batch write requests. Any other operations on this class will defer to the underlying boto3 DynamoDB Table's implementation and will not be encrypted/decrypted. - - Note: The update_item operation is not currently supported. Calling this operation will raise NotImplementedError. """ def __init__( @@ -60,7 +61,7 @@ def __init__( encryption_config: DynamoDbTablesEncryptionConfig, ): """ - Create an EncryptedTable object. + Create an ``EncryptedTable`` object. Args: table (ServiceResource): Initialized boto3 DynamoDB table @@ -79,15 +80,16 @@ def put_item(self, **kwargs) -> dict[str, Any]: """ Put a single item to the table. Encrypts the item before writing to DynamoDB. - The parameters and return value match the boto3 DynamoDB table put_item API: + The input and output syntaxes match those for the boto3 DynamoDB table ``put_item`` API: + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/put_item.html Args: - **kwargs: Keyword arguments to pass to the operation. These match the boto3 put_item API parameters. - The "Item" field will be encrypted locally before being written to DynamoDB. + **kwargs: Keyword arguments to pass to the operation. This matches the boto3 Table ``put_item`` request + syntax. The value in ``"Item"`` will be encrypted locally before being written to DynamoDB. Returns: - dict: The response from DynamoDB. This matches the boto3 put_item API response. + dict: The response from DynamoDB. This matches the boto3 ``put_item`` response syntax. """ return self._table_operation_logic( @@ -107,15 +109,17 @@ def get_item(self, **kwargs) -> dict[str, Any]: """ Get a single item from the table. Decrypts the item after reading from DynamoDB. - The parameters and return value match the boto3 DynamoDB table get_item API: + The input and output syntaxes match those for the boto3 DynamoDB table ``get_item`` API: + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/get_item.html Args: - **kwargs: Keyword arguments to pass to the operation. These match the boto3 get_item API parameters. + **kwargs: Keyword arguments to pass to the operation. This matches the boto3 Table ``get_item`` request + syntax. Returns: - dict: The response from DynamoDB. This matches the boto3 get_item API response. - The "Item" field will be decrypted locally after being read from DynamoDB. + dict: The response from DynamoDB. This matches the boto3 Table ``get_item`` response syntax. + The value in ``"Item"`` will be decrypted locally after being read from DynamoDB. """ return self._table_operation_logic( @@ -135,15 +139,17 @@ def query(self, **kwargs) -> dict[str, Any]: """ Query items from the table or index. Decrypts any returned items. - The parameters and return value match the boto3 DynamoDB table query API: + The input and output syntaxes match those for the boto3 DynamoDB table ``query`` API: + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/query.html Args: - **kwargs: Keyword arguments to pass to the operation. These match the boto3 query API parameters. + **kwargs: Keyword arguments to pass to the operation. This matches the boto3 Table ``query`` request + syntax. Returns: - dict: The response from DynamoDB. This matches the boto3 query API response. - The "Items" field will be decrypted locally after being read from DynamoDB. + dict: The response from DynamoDB. This matches the boto3 Table ``query`` response syntax. + The value in ``"Items"`` will be decrypted locally after being read from DynamoDB. """ return self._table_operation_logic( @@ -163,15 +169,17 @@ def scan(self, **kwargs) -> dict[str, Any]: """ Scan the entire table or index. Decrypts any returned items. - The parameters and return value match the boto3 DynamoDB table scan API: + The input and output syntaxes match those for the boto3 DynamoDB table ``scan`` API: + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/scan.html Args: - **kwargs: Keyword arguments to pass to the operation. These match the boto3 scan API parameters. + **kwargs: Keyword arguments to pass to the operation. This matches the boto3 Table ``scan`` request + syntax. Returns: - dict: The response from DynamoDB. This matches the boto3 scan API response. - The "Items" field will be decrypted locally after being read from DynamoDB. + dict: The response from DynamoDB. This matches the boto3 Table ``scan`` response syntax. + The value in ``"Items"`` will be decrypted locally after being read from DynamoDB. """ return self._table_operation_logic( @@ -189,7 +197,7 @@ def scan(self, **kwargs) -> dict[str, Any]: def update_item(self, **kwargs): """ - Not implemented. Raises NotImplementedError. + Not implemented. Raises ``NotImplementedError``. Args: **kwargs: Any arguments passed to this method diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py index 4ad797c38..38a58f95a 100644 --- a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py @@ -35,6 +35,10 @@ def plaintext_table(): return table +# Creates a matrix of tests for each value in param, +# with a user-friendly string for test output: +# encrypted = True -> "encrypted" +# encrypted = False -> "plaintext" @pytest.fixture(params=[True, False], ids=["encrypted", "plaintext"]) def encrypted(request): return request.param @@ -52,6 +56,10 @@ def table(encrypted): return plaintext_table() +# Creates a matrix of tests for each value in param, +# with a user-friendly string for test output: +# use_complex_item = True -> "complex_item" +# use_complex_item = False -> "simple_item" @pytest.fixture(params=[simple_item_dict, complex_item_dict], ids=["simple_item", "complex_item"]) def test_item(request): return request.param @@ -135,24 +143,13 @@ def test_GIVEN_items_in_table_WHEN_query_THEN_items_are_decrypted_correctly(tabl assert len(query_response["Items"]) == 1 assert query_response["Items"][0] == put_item_request_dict["Item"] - # Scans work, but the test items are not found because - # DDB only returns the first 1MB of data, and the test items - # are not in the first 1MB sometimes. We probably need a new table. - # TODO: Add a new table for these tests, enable tests. - # # When: Scanning with filter that matches only our test items - # scan_response = encrypted_table.scan(**scan_request_dict) - # # Then: Scan returns both test items - # assert scan_response["ResponseMetadata"]["HTTPStatusCode"] == 200 - # assert len(scan_response["Items"]) == 2 - # # Check each test item is found in scan results - # found_items = scan_response["Items"] - # assert all(any(found_item == item for found_item in found_items) for item in items) - @pytest.fixture def scan_request(encrypted, test_item): if encrypted: request = basic_scan_request_dict(test_item) + # If the encrypted scan encounters a plaintext item, the scan will fail. + # To avoid this, encrypted scans add a filter expression that matches only encrypted items. request["FilterExpression"] = request["FilterExpression"] + " AND attribute_exists (#sig)" request["ExpressionAttributeNames"] = {} request["ExpressionAttributeNames"]["#sig"] = "amzn-ddb-map-sig" diff --git a/TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBTable.py b/TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBTable.py index e27df97f5..469b9ae15 100644 --- a/TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBTable.py +++ b/TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBTable.py @@ -12,21 +12,13 @@ from smithy_dafny_standard_library.internaldafny.generated import Wrappers from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb.errors import _smithy_error_to_dafny_error from aws_dbesdk_dynamodb_test_vectors.waiting_boto3_ddb_client import WaitingLocalDynamoClient -from aws_dbesdk_dynamodb.transform import ( - dict_to_ddb, - ddb_to_dict, -) -from aws_dbesdk_dynamodb.internal import client_to_resource - -from boto3.dynamodb.conditions import Key, Attr, And, Or, Not, Contains -from boto3.dynamodb.types import TypeDeserializer -# from .....test.resource_formatted_queries import (queries, complex_queries) +from boto3.dynamodb.conditions import Key, Attr # When querying, DBESDK DDB TestVectors will pass the Table the query as a string. # The Table could accept this string as-is and process it correctly. # However, EncryptedTables have extra logic to process boto3 Conditions. -# I want to test this extra logic as much as possible. +# This extra logic should be tested as much as possible. # This map converts some known query strings to equivalent Conditions. # TestVectors will pass the query string (map key) to the Table; # the Table's internal logic will look up the query string in this map: @@ -34,8 +26,8 @@ # - Not found: Query with original string. Table accepts strings. # This map contains all query strings in the TestVectors' data.json as of commit # 4f18689f79243c9a5ab0f3a23108671defddeac4 -# If any query strings are added to TestVectors, they COULD be added here, -# but do not need to be added. +# If any query strings are added to TestVectors, they COULD be added here; +# if they are not added, the Table will accept the string as-is. known_query_string_to_condition_map = { # "Basic" queries "RecNum = :zero": Key("RecNum").eq(":zero"), @@ -130,9 +122,9 @@ def get_item(self, **kwargs): return client_output def batch_write_item(self, **kwargs): - # There isn't a resource shape for this; + # The table doesn't support batch_write_item, but supports batch_writer. + # Translate the batch_write_item request to batch_writer requests. table_input = self._client_shape_to_resource_shape_converter.batch_write_item_request(kwargs) - # table_output = self._table.batch_write_item(**table_input) with self._table.batch_writer() as batch_writer: for _, items in table_input["RequestItems"].items(): for item in items: @@ -142,7 +134,7 @@ def batch_write_item(self, **kwargs): batch_writer.delete_item(item["DeleteRequest"]["Key"]) else: raise ValueError(f"Unknown request type: {item}") - # There isn't a shape for the output, but luckily the output can be an empty dict: + # An empty dict is valid output: # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/client/batch_write_item.html client_output = {} return client_output @@ -158,8 +150,7 @@ def scan(self, **kwargs): if "KeyConditionExpression" in table_input: if table_input["KeyConditionExpression"] in known_query_string_to_condition_map: # Turn the query into the resource-formatted query - query = known_query_string_to_condition_map[table_input["KeyConditionExpression"]] - table_input["KeyConditionExpression"] = query + table_input["KeyConditionExpression"] = known_query_string_to_condition_map[table_input["KeyConditionExpression"]] if "FilterExpression" in table_input: if table_input["FilterExpression"] in known_query_string_to_condition_map: # Turn the query into the resource-formatted query @@ -182,8 +173,7 @@ def query(self, **kwargs): if "KeyConditionExpression" in table_input: if table_input["KeyConditionExpression"] in known_query_string_to_condition_map: # Turn the query into the resource-formatted query - query = known_query_string_to_condition_map[table_input["KeyConditionExpression"]] - table_input["KeyConditionExpression"] = query + table_input["KeyConditionExpression"] = known_query_string_to_condition_map[table_input["KeyConditionExpression"]] if "FilterExpression" in table_input: if table_input["FilterExpression"] in known_query_string_to_condition_map: # Turn the query into the resource-formatted query From 042e4b8464e4de5fd18a027aee462d4a06e8cf8d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 14 May 2025 14:23:41 -0700 Subject: [PATCH 3/9] sync --- .../src/aws_dbesdk_dynamodb/encrypted/table.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py index d4a88e454..bdf22da47 100644 --- a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """High-level helper class to provide an encrypting wrapper for boto3 DynamoDB tables.""" from collections.abc import Callable +from copy import deepcopy from typing import Any from boto3.dynamodb.table import BatchWriter @@ -38,9 +39,10 @@ class EncryptedTable(EncryptedBotoInterface): The API matches the standard boto3 DynamoDB table interface: - https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table.html + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/index.html This class will encrypt/decrypt items for the following operations: + * ``put_item`` * ``get_item`` * ``query`` @@ -51,7 +53,7 @@ class EncryptedTable(EncryptedBotoInterface): Calling ``batch_writer()`` will return a ``BatchWriter`` that transparently encrypts batch write requests. Any other operations on this class will defer to the underlying boto3 DynamoDB Table's implementation - and will not be encrypted/decrypted. + and will not be encrypted/decrypted. """ def __init__( @@ -119,7 +121,7 @@ def get_item(self, **kwargs) -> dict[str, Any]: Returns: dict: The response from DynamoDB. This matches the boto3 Table ``get_item`` response syntax. - The value in ``"Item"`` will be decrypted locally after being read from DynamoDB. + The value in ``"Item"`` will be decrypted locally after being read from DynamoDB. """ return self._table_operation_logic( @@ -149,7 +151,7 @@ def query(self, **kwargs) -> dict[str, Any]: Returns: dict: The response from DynamoDB. This matches the boto3 Table ``query`` response syntax. - The value in ``"Items"`` will be decrypted locally after being read from DynamoDB. + The value in ``"Items"`` will be decrypted locally after being read from DynamoDB. """ return self._table_operation_logic( @@ -179,7 +181,7 @@ def scan(self, **kwargs) -> dict[str, Any]: Returns: dict: The response from DynamoDB. This matches the boto3 Table ``scan`` response syntax. - The value in ``"Items"`` will be decrypted locally after being read from DynamoDB. + The value in ``"Items"`` will be decrypted locally after being read from DynamoDB. """ return self._table_operation_logic( @@ -268,9 +270,10 @@ def _table_operation_logic( dict: The transformed response from DynamoDB """ + table_input = deepcopy(operation_input) # EncryptedTable inputs are formatted as standard dictionaries, but DBESDK transformations expect DynamoDB JSON. # Convert from standard dictionaries to DynamoDB JSON. - input_transform_input = input_resource_to_client_shape_transform_method(operation_input) + input_transform_input = input_resource_to_client_shape_transform_method(table_input) # Apply DBESDK transformation to the input input_transform_output = input_encryption_transform_method( From 9aa4a4808a30b1c6028fcbe74e460ce80322267f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 14 May 2025 16:29:38 -0700 Subject: [PATCH 4/9] sync --- .../runtimes/python/test/integ/encrypted/test_table.py | 4 ---- .../src/basic_put_get_example/with_encrypted_table.py | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py index 38a58f95a..f5785b8eb 100644 --- a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py @@ -1,6 +1,5 @@ import boto3 import pytest -from boto3.dynamodb.types import TypeDeserializer, TypeSerializer from aws_dbesdk_dynamodb.encrypted.table import EncryptedTable @@ -16,9 +15,6 @@ basic_scan_request_dict, ) -serializer = TypeSerializer() -deserializer = TypeDeserializer() - def encrypted_table(): """Create an encrypted table.""" diff --git a/Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/with_encrypted_table.py b/Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/with_encrypted_table.py index d038e0dba..396558a86 100644 --- a/Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/with_encrypted_table.py +++ b/Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/with_encrypted_table.py @@ -20,11 +20,11 @@ ) from aws_cryptographic_material_providers.mpl.references import IKeyring from aws_dbesdk_dynamodb.encrypted.table import EncryptedTable -from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb.models import ( +from aws_dbesdk_dynamodb.structures.dynamodb import ( DynamoDbTableEncryptionConfig, DynamoDbTablesEncryptionConfig, ) -from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_structuredencryption.models import ( +from aws_dbesdk_dynamodb.structures.structured_encryption import ( CryptoAction, ) From 28e03f332967cbcf11e42cbf00491ad4d6c82f57 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 19 May 2025 15:23:01 -0700 Subject: [PATCH 5/9] sync --- .../aws_dbesdk_dynamodb/encrypted/table.py | 67 +++++++++++++-- .../python/test/integ/encrypted/test_table.py | 85 ++++++++++++++----- .../with_encrypted_table.py | 5 +- .../test_with_encrypted_table.py | 4 +- 4 files changed, 129 insertions(+), 32 deletions(-) diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py index bdf22da47..8cc1994d1 100644 --- a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py @@ -19,6 +19,8 @@ DynamoDbEncryptionTransforms, ) from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb_transforms.models import ( + DeleteItemInputTransformInput, + DeleteItemOutputTransformInput, GetItemInputTransformInput, GetItemOutputTransformInput, PutItemInputTransformInput, @@ -27,6 +29,8 @@ QueryOutputTransformInput, ScanInputTransformInput, ScanOutputTransformInput, + UpdateItemInputTransformInput, + UpdateItemOutputTransformInput, ) @@ -47,8 +51,10 @@ class EncryptedTable(EncryptedBotoInterface): * ``get_item`` * ``query`` * ``scan`` + * ``delete_item`` - The ``update_item`` operation is not currently supported. Calling this operation will raise ``NotImplementedError``. + Any calls to ``update_item`` can only update unsigned attributes. If an attribute to be updated is marked as signed, + this operation will raise a ``DynamoDbEncryptionTransformsException``. Calling ``batch_writer()`` will return a ``BatchWriter`` that transparently encrypts batch write requests. @@ -197,18 +203,69 @@ def scan(self, **kwargs) -> dict[str, Any]: table_method=self._table.scan, ) + def delete_item(self, **kwargs) -> dict[str, Any]: + """ + Delete an item from the table. + + The input and output syntaxes match those for the boto3 DynamoDB table ``delete_item`` API: + + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/delete_item.html + + Args: + **kwargs: Keyword arguments to pass to the operation. This matches the boto3 Table ``delete_item`` request + syntax. + + Returns: + dict: The response from DynamoDB. This matches the boto3 Table ``delete_item`` response syntax. + Any values in ``"Attributes"`` will be decrypted locally after being read from DynamoDB. + + """ + return self._table_operation_logic( + operation_input=kwargs, + input_encryption_transform_method=self._transformer.delete_item_input_transform, + input_encryption_transform_shape=DeleteItemInputTransformInput, + input_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.delete_item_request, + input_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.delete_item_request, + output_encryption_transform_method=self._transformer.delete_item_output_transform, + output_encryption_transform_shape=DeleteItemOutputTransformInput, + output_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.delete_item_response, + output_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.delete_item_response, + table_method=self._table.delete_item, + ) + def update_item(self, **kwargs): """ - Not implemented. Raises ``NotImplementedError``. + Update an unsigned attribute in the table. + + If the attribute is signed, this operation will raise DynamoDbEncryptionTransformsException. + + The input and output syntaxes match those for the boto3 DynamoDB table ``update_item`` API: + + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/update_item.html Args: - **kwargs: Any arguments passed to this method + **kwargs: Keyword arguments to pass to the operation. This matches the boto3 Table ``update_item`` request + syntax. + + Returns: + dict: The response from DynamoDB. This matches the boto3 Table ``update_item`` response syntax. Raises: - NotImplementedError: This operation is not yet implemented + DynamoDbEncryptionTransformsException: If an attribute specified in the ``UpdateExpression`` is signed. """ - raise NotImplementedError('"update_item" is not yet implemented') + return self._table_operation_logic( + operation_input=kwargs, + input_encryption_transform_method=self._transformer.update_item_input_transform, + input_encryption_transform_shape=UpdateItemInputTransformInput, + input_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.update_item_request, + input_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.update_item_request, + output_encryption_transform_method=self._transformer.update_item_output_transform, + output_encryption_transform_shape=UpdateItemOutputTransformInput, + output_resource_to_client_shape_transform_method=self._resource_shape_to_client_shape_converter.update_item_response, + output_client_to_resource_shape_transform_method=self._client_shape_to_resource_shape_converter.update_item_response, + table_method=self._table.update_item, + ) def batch_writer(self, overwrite_by_pkeys: list[str] | None = None) -> BatchWriter: """ diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py index f5785b8eb..8d8a86883 100644 --- a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py @@ -2,6 +2,9 @@ import pytest from aws_dbesdk_dynamodb.encrypted.table import EncryptedTable +from aws_dbesdk_dynamodb.smithygenerated.aws_cryptography_dbencryptionsdk_dynamodb_transforms.errors import ( + DynamoDbEncryptionTransformsException, +) from ...constants import ( INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, @@ -9,10 +12,13 @@ ) from ...items import complex_item_dict, simple_item_dict from ...requests import ( + basic_delete_item_request_dict, basic_get_item_request_dict, basic_put_item_request_dict, basic_query_request_dict, basic_scan_request_dict, + basic_update_item_request_dict_signed_attribute, + basic_update_item_request_dict_unsigned_attribute, ) @@ -61,21 +67,37 @@ def test_item(request): return request.param -def test_GIVEN_item_WHEN_basic_put_and_basic_get_THEN_round_trip_passes(table, test_item): - """Test put_item and get_item operations.""" - # Given: Simple and complex items in appropriate format for client +def test_GIVEN_item_WHEN_basic_put_AND_basic_get_AND_basic_delete_THEN_round_trip_passes(table, test_item): + """Test put_item, get_item, and delete_item operations.""" + # Given: Valid put_item request put_item_request_dict = basic_put_item_request_dict(test_item) - - # When: Putting and getting item + # When: put_item put_response = table.put_item(**put_item_request_dict) + # Then: put_item succeeds assert put_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + # Given: Valid get_item request for the same item get_item_request_dict = basic_get_item_request_dict(test_item) + # When: get_item get_response = table.get_item(**get_item_request_dict) # Then: Simple item is encrypted and decrypted correctly assert get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 assert get_response["Item"] == put_item_request_dict["Item"] + # Given: Valid delete_item request for the same item + delete_item_request_dict = basic_delete_item_request_dict(test_item) + # When: delete_item + delete_response = table.delete_item(**delete_item_request_dict) + # Then: delete_item succeeds + assert delete_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Given: Valid get_item request for the same item + get_item_request_dict = basic_get_item_request_dict(test_item) + # When: get_item + get_response = table.get_item(**get_item_request_dict) + # Then: get_item is empty (i.e. the item was deleted) + assert "Item" not in get_response + def test_GIVEN_items_WHEN_batch_write_and_get_THEN_round_trip_passes( table, @@ -154,7 +176,6 @@ def scan_request(encrypted, test_item): def test_GIVEN_valid_put_and_scan_requests_WHEN_put_and_scan_THEN_round_trip_passes(table, test_item, scan_request): - """Test put_item and scan operations.""" # Given: Simple and complex items in appropriate format for client put_item_request_dict = basic_put_item_request_dict(test_item) table.put_item(**put_item_request_dict) @@ -166,24 +187,44 @@ def test_GIVEN_valid_put_and_scan_requests_WHEN_put_and_scan_THEN_round_trip_pas assert scan_response["ResponseMetadata"]["HTTPStatusCode"] == 200 -def test_WHEN_update_item_THEN_raises_not_implemented_error(): - # Given: Encrypted client and update item parameters +def test_GIVEN_update_for_unsigned_attribute_WHEN_update_item_THEN_passes(table, test_item): + # Given: some item is already in the table + put_response = table.put_item(**basic_put_item_request_dict(test_item)) + assert put_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Given: Valid update item request for unsigned attribute + update_item_request = basic_update_item_request_dict_unsigned_attribute(test_item) + # When: Calling update_item - with pytest.raises(NotImplementedError): - encrypted_table().update_item( - TableName=INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, - Key={"partition_key": "test-key", "sort_key": 1}, - UpdateExpression="SET attribute1 = :val", - ExpressionAttributeValues={":val": {"S": "new value"}}, - ) - # Then: NotImplementedError is raised + update_response = table.update_item(**update_item_request) + # Then: update_item succeeds + assert update_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + +def test_GIVEN_update_for_signed_attribute_WHEN_update_item_THEN_raises_DynamoDbEncryptionTransformsException( + table, test_item, encrypted +): + if not encrypted: + pytest.skip("Skipping negative test for plaintext client") + + # Given: some item is already in the table + put_response = table.put_item(**basic_put_item_request_dict(test_item)) + assert put_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Given: Valid update item request for signed attribute + update_item_request = basic_update_item_request_dict_signed_attribute(test_item) + + # Then: raises DynamoDbEncryptionTransformsException + with pytest.raises(DynamoDbEncryptionTransformsException): + # When: Calling update_item + table.update_item(**update_item_request) def test_WHEN_call_passthrough_method_THEN_correct_response_is_returned(): """Test that calling a passthrough method returns the correct response.""" - # Given: Encrypted client - # When: Calling some passthrough method that does not explicitly exist on EncryptedClient, - # but exists on the underlying boto3 client - response = encrypted_table().meta.client.list_backups() - # Then: Correct response is returned, i.e. EncryptedClient forwards the call to the underlying boto3 client - assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + # Given: Encrypted or plaintext table + # When: Calling some passthrough method that does not explicitly exist on EncryptedTable, + # but exists on the underlying boto3 table + response = encrypted_table().table_name + # Then: Correct response is returned, i.e. EncryptedTable forwards the call to the underlying boto3 table + assert response == INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME diff --git a/Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/with_encrypted_table.py b/Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/with_encrypted_table.py index 396558a86..161f90a16 100644 --- a/Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/with_encrypted_table.py +++ b/Examples/runtimes/python/DynamoDBEncryption/src/basic_put_get_example/with_encrypted_table.py @@ -110,7 +110,7 @@ def encrypted_table_put_get_example( table_configs[dynamodb_table_name] = table_config tables_config = DynamoDbTablesEncryptionConfig(table_encryption_configs=table_configs) - # 5. Create the EncryptedClient + # 5. Create the EncryptedTable encrypted_table = EncryptedTable( table=boto3.resource("dynamodb").Table(dynamodb_table_name), encryption_config=tables_config, @@ -128,7 +128,6 @@ def encrypted_table_put_get_example( } put_item_request = { - "TableName": dynamodb_table_name, "Item": item_to_encrypt, } @@ -142,7 +141,7 @@ def encrypted_table_put_get_example( # back the original item. key_to_get = {"partition_key": "BasicPutGetExample", "sort_key": 0} - get_item_request = {"TableName": dynamodb_table_name, "Key": key_to_get} + get_item_request = {"Key": key_to_get} get_item_response = encrypted_table.get_item(**get_item_request) diff --git a/Examples/runtimes/python/DynamoDBEncryption/test/basic_put_get_example/test_with_encrypted_table.py b/Examples/runtimes/python/DynamoDBEncryption/test/basic_put_get_example/test_with_encrypted_table.py index 5a5c665de..5b30d7f45 100644 --- a/Examples/runtimes/python/DynamoDBEncryption/test/basic_put_get_example/test_with_encrypted_table.py +++ b/Examples/runtimes/python/DynamoDBEncryption/test/basic_put_get_example/test_with_encrypted_table.py @@ -1,6 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Test suite for the EncryptedClient example.""" +"""Test suite for the EncryptedTable example.""" import pytest from ...src.basic_put_get_example.with_encrypted_table import encrypted_table_put_get_example @@ -9,7 +9,7 @@ def test_encrypted_table_put_get_example(): - """Test function for encrypt and decrypt using the EncryptedClient example.""" + """Test function for encrypt and decrypt using the EncryptedTable example.""" test_kms_key_id = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" test_dynamodb_table_name = "DynamoDbEncryptionInterceptorTestTable" encrypted_table_put_get_example(test_kms_key_id, test_dynamodb_table_name) From 168efcc00909a60342429b633100de7aff07f6c3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 May 2025 09:12:19 -0700 Subject: [PATCH 6/9] sync --- TestVectors/runtimes/python/test/table/__init__.py | 3 +++ TestVectors/runtimes/python/test/table/test_dafny_wrapper.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/TestVectors/runtimes/python/test/table/__init__.py b/TestVectors/runtimes/python/test/table/__init__.py index e69de29bb..fa977e22f 100644 --- a/TestVectors/runtimes/python/test/table/__init__.py +++ b/TestVectors/runtimes/python/test/table/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub to allow relative imports of examples from tests.""" diff --git a/TestVectors/runtimes/python/test/table/test_dafny_wrapper.py b/TestVectors/runtimes/python/test/table/test_dafny_wrapper.py index c21c59a58..390937d9a 100644 --- a/TestVectors/runtimes/python/test/table/test_dafny_wrapper.py +++ b/TestVectors/runtimes/python/test/table/test_dafny_wrapper.py @@ -16,6 +16,9 @@ sys.path.append(test_dir + "/internaldafny/extern") sys.path.append(test_dir + "/internaldafny/generated") +# These imports set up the tests to use: +# - An EncryptedTable with a shim to make it appear to Dafny-generated code as a DBESDK client +# - A DictItemEncryptor with a shim to take in DDB-formatted JSON and return DDB-formatted JSON import aws_dbesdk_dynamodb_test_vectors.internaldafny.extern.CreateInterceptedDDBTable import aws_dbesdk_dynamodb_test_vectors.internaldafny.extern.CreateWrappedDictItemEncryptor @@ -25,6 +28,7 @@ # - get_item # - query # - scan +# - update_item # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/index.html#DynamoDB.Table # # Unsupported operations on Tables are that are supported by DBESDK are: From 38979de4c9502bb48e6f2e85171c95b565d0ba0c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 May 2025 09:25:05 -0700 Subject: [PATCH 7/9] sync --- .../runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py | 3 +++ TestVectors/runtimes/python/test/__init__.py | 1 + 2 files changed, 4 insertions(+) diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py index 8cc1994d1..d684d3e5f 100644 --- a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py @@ -364,6 +364,9 @@ def _table_operation_logic( # Copy any missing fields from the SDK output to the response (e.g. `ConsumedCapacity`) dbesdk_response = self._copy_sdk_response_to_dbesdk_response(sdk_output, dbesdk_response) + # Clean up the expression builder for the next operation + self._resource_shape_to_client_shape_converter.expression_builder.reset() + return dbesdk_response @property diff --git a/TestVectors/runtimes/python/test/__init__.py b/TestVectors/runtimes/python/test/__init__.py index f94fd12a2..fa977e22f 100644 --- a/TestVectors/runtimes/python/test/__init__.py +++ b/TestVectors/runtimes/python/test/__init__.py @@ -1,2 +1,3 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +"""Stub to allow relative imports of examples from tests.""" From 91b1e53b9bb1aebf25c67ef274bb64c0c1a2dfc7 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 29 May 2025 13:21:25 -0700 Subject: [PATCH 8/9] sync --- .../aws_dbesdk_dynamodb/encrypted/table.py | 1 + .../python/test/integ/encrypted/test_table.py | 98 ++++++---- .../extern/CreateInterceptedDDBTable.py | 168 +++++++++++++----- 3 files changed, 190 insertions(+), 77 deletions(-) diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py index d684d3e5f..45ca61ceb 100644 --- a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/encrypted/table.py @@ -328,6 +328,7 @@ def _table_operation_logic( """ table_input = deepcopy(operation_input) + # EncryptedTable inputs are formatted as standard dictionaries, but DBESDK transformations expect DynamoDB JSON. # Convert from standard dictionaries to DynamoDB JSON. input_transform_input = input_resource_to_client_shape_transform_method(table_input) diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py index 8d8a86883..48bd3f0da 100644 --- a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py @@ -1,3 +1,6 @@ +import uuid +from copy import deepcopy + import boto3 import pytest @@ -10,7 +13,7 @@ INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, INTEG_TEST_DEFAULT_TABLE_CONFIGS, ) -from ...items import complex_item_dict, simple_item_dict +from ...items import complex_item_dict, complex_key_dict, simple_item_dict, simple_key_dict from ...requests import ( basic_delete_item_request_dict, basic_get_item_request_dict, @@ -58,13 +61,20 @@ def table(encrypted): return plaintext_table() +@pytest.fixture(scope="module") +def test_run_suffix(): + return "-" + str(uuid.uuid4()) + + # Creates a matrix of tests for each value in param, # with a user-friendly string for test output: # use_complex_item = True -> "complex_item" # use_complex_item = False -> "simple_item" @pytest.fixture(params=[simple_item_dict, complex_item_dict], ids=["simple_item", "complex_item"]) -def test_item(request): - return request.param +def test_item(request, test_run_suffix): + item = deepcopy(request.param) + item["partition_key"] += test_run_suffix + return item def test_GIVEN_item_WHEN_basic_put_AND_basic_get_AND_basic_delete_THEN_round_trip_passes(table, test_item): @@ -99,51 +109,57 @@ def test_GIVEN_item_WHEN_basic_put_AND_basic_get_AND_basic_delete_THEN_round_tri assert "Item" not in get_response +@pytest.fixture +def multiple_test_items(test_run_suffix): + """Get two test items in the appropriate format for the client.""" + items = [deepcopy(simple_item_dict), deepcopy(complex_item_dict)] + for item in items: + item["partition_key"] += test_run_suffix + return items + + +@pytest.fixture +def multiple_test_keys(test_run_suffix): + """Get two test keys in the appropriate format for the client.""" + keys = [deepcopy(simple_key_dict), deepcopy(complex_key_dict)] + for key in keys: + key["partition_key"] += test_run_suffix + return keys + + def test_GIVEN_items_WHEN_batch_write_and_get_THEN_round_trip_passes( table, + multiple_test_items, + multiple_test_keys, ): # Given: Simple and complex items in appropriate format for client # When: Batch put items with table.batch_writer() as batch_writer: # boto3 documentation for batch_writer.put_item() is incorrect; # the method accepts the item directly, not the item inside an "Item" key. - batch_writer.put_item(simple_item_dict) - batch_writer.put_item(complex_item_dict) + for item in multiple_test_items: + batch_writer.put_item(item) # When: Get items - get_item_request_dict = basic_get_item_request_dict(simple_item_dict) - get_response = table.get_item(**get_item_request_dict) - # Then: All items are encrypted and decrypted correctly - assert get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 - assert get_response["Item"] == simple_item_dict - - get_item_request_dict = basic_get_item_request_dict(complex_item_dict) - get_response = table.get_item(**get_item_request_dict) - # Then: All items are encrypted and decrypted correctly - assert get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 - assert get_response["Item"] == complex_item_dict + for item in multiple_test_items: + get_item_request_dict = basic_get_item_request_dict(item) + get_response = table.get_item(**get_item_request_dict) + # Then: All items are encrypted and decrypted correctly + assert get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert get_response["Item"] == item # When: Batch delete items with table.batch_writer() as batch_writer: - batch_writer.delete_item( - {"partition_key": simple_item_dict["partition_key"], "sort_key": simple_item_dict["sort_key"]} - ) - batch_writer.delete_item( - {"partition_key": complex_item_dict["partition_key"], "sort_key": complex_item_dict["sort_key"]} - ) + for key in multiple_test_keys: + batch_writer.delete_item(key) # When: Get items - get_item_request_dict = basic_get_item_request_dict(simple_item_dict) - get_response = table.get_item(**get_item_request_dict) - # Then: All items are encrypted and decrypted correctly - assert get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 - assert "Item" not in get_response - - get_item_request_dict = basic_get_item_request_dict(complex_item_dict) - get_response = table.get_item(**get_item_request_dict) - # Then: All items are encrypted and decrypted correctly - assert get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 - assert "Item" not in get_response + for item in multiple_test_items: + get_item_request_dict = basic_get_item_request_dict(item) + get_response = table.get_item(**get_item_request_dict) + # Then: All items are encrypted and decrypted correctly + assert get_response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert "Item" not in get_response def test_GIVEN_items_in_table_WHEN_query_THEN_items_are_decrypted_correctly(table, test_item): @@ -183,7 +199,10 @@ def test_GIVEN_valid_put_and_scan_requests_WHEN_put_and_scan_THEN_round_trip_pas # When: Scanning items scan_request_dict = scan_request scan_response = table.scan(**scan_request_dict) - # Then: Scan returns both test items + # Then: Scan succeeds + # Can't assert anything about the scan; + # there are too many items. + # The critical assertion is that the scan succeeds. assert scan_response["ResponseMetadata"]["HTTPStatusCode"] == 200 @@ -228,3 +247,14 @@ def test_WHEN_call_passthrough_method_THEN_correct_response_is_returned(): response = encrypted_table().table_name # Then: Correct response is returned, i.e. EncryptedTable forwards the call to the underlying boto3 table assert response == INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME + + +# Delete the items in the table after the module runs +@pytest.fixture(scope="module", autouse=True) +def cleanup_after_module(test_run_suffix): + yield + table = boto3.resource("dynamodb").Table(INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME) + items = [deepcopy(simple_item_dict), deepcopy(complex_item_dict)] + for item in items: + item["partition_key"] = item["partition_key"] + test_run_suffix + table.delete_item(**basic_delete_item_request_dict(item)) diff --git a/TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBTable.py b/TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBTable.py index 469b9ae15..852b5b5d2 100644 --- a/TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBTable.py +++ b/TestVectors/runtimes/python/src/aws_dbesdk_dynamodb_test_vectors/internaldafny/extern/CreateInterceptedDDBTable.py @@ -14,6 +14,51 @@ from aws_dbesdk_dynamodb_test_vectors.waiting_boto3_ddb_client import WaitingLocalDynamoClient from boto3.dynamodb.conditions import Key, Attr +from decimal import Decimal + +import json +import os +from typing import Any, Dict + +def load_test_data() -> Dict[str, Any]: + """Load the ExpressionAttributeValues from data.json file.""" + # Get the directory of the current file + current_dir = os.getcwd() + # Navigate to the data.json file + data_file = os.path.join(current_dir, 'data.json') + + with open(data_file, 'r') as f: + return json.load(f)["Values"] + +expression_attribute_values_from_json = load_test_data() + +def get_test_value(name) -> Any: + """ + Get a test value from the Values section of data.json. + + Args: + name: The name of the value to retrieve (e.g. ":zero", ":one", etc.) + + Returns: + The value from the Values section + + Raises: + KeyError: If the requested value name is not found + """ + if name not in expression_attribute_values_from_json: + raise KeyError(f"Value '{name}' not found in test data") + value = expression_attribute_values_from_json[name] + if isinstance(value, dict): + if "N" in value: + return Decimal(value["N"]) + elif "SS" in value: + return set(value["SS"]) + elif "L" in value: + return list(value["L"]) + else: + raise KeyError(f"Unknown ExpressionAttributeValue type: {value}") + return value + # When querying, DBESDK DDB TestVectors will pass the Table the query as a string. # The Table could accept this string as-is and process it correctly. @@ -28,32 +73,29 @@ # 4f18689f79243c9a5ab0f3a23108671defddeac4 # If any query strings are added to TestVectors, they COULD be added here; # if they are not added, the Table will accept the string as-is. -known_query_string_to_condition_map = { +known_filter_expression_string_to_condition_map = { # "Basic" queries - "RecNum = :zero": Key("RecNum").eq(":zero"), - "RecNum = :one": Key("RecNum").eq(":one"), - "RecNum = :zero": Attr("RecNum").eq(":zero"), - "RecNum <= :zero": Attr("RecNum").lte(":zero"), - "RecNum > :zero": Attr("RecNum").gt(":zero"), - "RecNum >= :zero": Attr("RecNum").gte(":zero"), - "RecNum <> :zero": Attr("RecNum").ne(":zero"), - "RecNum = :zero": Attr("RecNum").eq(":zero"), - "RecNum = :one": Attr("RecNum").eq(":one"), - "Nine between :zeroD and :three": Attr("Nine").between(":zeroD", ":three"), - "Nine between :nineD and :nine": Attr("Nine").between(":nineD", ":nine"), - "Nine between :nine and :three": Attr("Nine").between(":nine", ":three"), - "Nine between :nine and :nine": Attr("Nine").between(":nine", ":nine"), - "NumberTest = :NumberTest": Attr("NumberTest").eq(":NumberTest"), - "RecNum in (:zero, :one)": Attr("RecNum").is_in([":zero", ":one"]), - "Two = :two": Attr("Two").eq(":two"), - "Two = :two or Three = :three or Four = :four OR Five = :five": Attr("Two").eq(":two") | Attr("Three").eq(":three") | Attr("Four").eq(":four") | Attr("Five").eq(":five"), - "Two = :two and Three = :three and Four = :four and Five = :five": Attr("Two").eq(":two") & Attr("Three").eq(":three") & Attr("Four").eq(":four") & Attr("Five").eq(":five"), - "Two in (:two, :three, :four, :five)": Attr("Two").is_in([":two", ":three", ":four", ":five"]), - "Five in (:two, :three, :four, :five)": Attr("Five").is_in([":two", ":three", ":four", ":five"]), - "Five in (:strset)": Attr("Five").is_in([":strset"]), - "Five in (:strlist)": Attr("Five").is_in([":strlist"]), - "contains(One, :oneA)": Attr("One").contains(":oneA"), - "contains(One, :oneB)": Attr("One").contains(":oneB"), + "RecNum = :zero": Attr("RecNum").eq(get_test_value(":zero")), + "RecNum <= :zero": Attr("RecNum").lte(get_test_value(":zero")), + "RecNum > :zero": Attr("RecNum").gt(get_test_value(":zero")), + "RecNum >= :zero": Attr("RecNum").gte(get_test_value(":zero")), + "RecNum <> :zero": Attr("RecNum").ne(get_test_value(":zero")), + "RecNum = :one": Attr("RecNum").eq(get_test_value(":one")), + "Nine between :zeroD and :three": Attr("Nine").between(get_test_value(":zeroD"), get_test_value(":three")), + "Nine between :nineD and :nine": Attr("Nine").between(get_test_value(":nineD"), get_test_value(":nine")), + "Nine between :nine and :three": Attr("Nine").between(get_test_value(":nine"), get_test_value(":three")), + "Nine between :nine and :nine": Attr("Nine").between(get_test_value(":nine"), get_test_value(":nine")), + "NumberTest = :NumberTest": Attr("NumberTest").eq(get_test_value(":NumberTest")), + "RecNum in (:zero, :one)": Attr("RecNum").is_in([get_test_value(":zero"), get_test_value(":one")]), + "Two = :two": Attr("Two").eq(get_test_value(":two")), + "Two = :two or Three = :three or Four = :four OR Five = :five": Attr("Two").eq(get_test_value(":two")) | Attr("Three").eq(get_test_value(":three")) | Attr("Four").eq(get_test_value(":four")) | Attr("Five").eq(get_test_value(":five")), + "Two = :two and Three = :three and Four = :four and Five = :five": Attr("Two").eq(get_test_value(":two")) & Attr("Three").eq(get_test_value(":three")) & Attr("Four").eq(get_test_value(":four")) & Attr("Five").eq(get_test_value(":five")), + "Two in (:two, :three, :four, :five)": Attr("Two").is_in([get_test_value(":two"), get_test_value(":three"), get_test_value(":four"), get_test_value(":five")]), + "Five in (:two, :three, :four, :five)": Attr("Five").is_in([get_test_value(":two"), get_test_value(":three"), get_test_value(":four"), get_test_value(":five")]), + "Five in (:strset)": Attr("Five").is_in([get_test_value(":strset")]), + "Five in (:strlist)": Attr("Five").is_in([get_test_value(":strlist")]), + "contains(One, :oneA)": Attr("One").contains(get_test_value(":oneA")), + "contains(One, :oneB)": Attr("One").contains(get_test_value(":oneB")), # Hard-coding returning the input string for these cases. # These conditions test undocumented behavior in DynamoDB that can't be expressed with boto3 Conditions. # The undocumented behavior is that `contains`' first parameter can be a value, @@ -73,14 +115,14 @@ "contains(:strset, One)": "contains(:strset, One)", # "Complex" queries - "Comp1 := :cmp1a": Attr("Comp1").eq(":cmp1a"), - "begins_with(Comp1, :cmp1c)": Attr("Comp1").begins_with(":cmp1c"), - "cmp1c < Comp1": Attr("cmp1c").lt("Comp1"), - "cmp1c = Comp1": Attr("cmp1c").eq("Comp1"), - "begins_with(Comp1, :cmp1d)": Attr("Comp1").begins_with(":cmp1d"), - "contains(Comp1, :cmp1c)": Attr("Comp1").contains(":cmp1c"), - "contains(Comp1, :cmp1d)": Attr("Comp1").contains(":cmp1d"), - "Comp1 = :cmp1b": Attr("Comp1").eq(":cmp1b"), + "Comp1 := :cmp1a": Attr("Comp1").eq(get_test_value(":cmp1a")), + "begins_with(Comp1, :cmp1c)": Attr("Comp1").begins_with(get_test_value(":cmp1c")), + "cmp1c < Comp1": Attr("cmp1c").lt(get_test_value(":cmp1c")), + "cmp1c = Comp1": Attr("cmp1c").eq(get_test_value(":cmp1c")), + "begins_with(Comp1, :cmp1d)": Attr("Comp1").begins_with(get_test_value(":cmp1d")), + "contains(Comp1, :cmp1c)": Attr("Comp1").contains(get_test_value(":cmp1c")), + "contains(Comp1, :cmp1d)": Attr("Comp1").contains(get_test_value(":cmp1d")), + "Comp1 = :cmp1b": Attr("Comp1").eq(get_test_value(":cmp1b")), # Another query that can't be translated to boto3 Conditions, # since attribute values aren't attribute names. @@ -88,6 +130,12 @@ ":cmp1c <= Comp1": ":cmp1c <= Comp1", } +# KeyConditionExpression strings expect Keys, not Attrs. +known_key_condition_expression_string_to_condition_map = { + "RecNum = :zero": Key("RecNum").eq(get_test_value(":zero")), + "RecNum = :one": Key("RecNum").eq(get_test_value(":one")), +} + class DynamoDBClientWrapperForDynamoDBTable: """ DBESDK TestVectors-internal wrapper class. @@ -148,13 +196,30 @@ def scan(self, **kwargs): # convert the string-based KeyConditionExpression and FilterExpression # into the boto3.conditions.Key and boto3.conditions.Attr resource-formatted queries. if "KeyConditionExpression" in table_input: - if table_input["KeyConditionExpression"] in known_query_string_to_condition_map: - # Turn the query into the resource-formatted query - table_input["KeyConditionExpression"] = known_query_string_to_condition_map[table_input["KeyConditionExpression"]] + if table_input["KeyConditionExpression"] in known_key_condition_expression_string_to_condition_map: + table_input["KeyConditionExpression"] = known_key_condition_expression_string_to_condition_map[table_input["KeyConditionExpression"]] + # boto3 Conditions cannot accept any externally-provided ExpressionAttributeValues + # if the KeyConditionExpression is not a string. + # If the KeyConditionExpression was replaced, remove the now-useless ExpressionAttributeValues. + if "ExpressionAttributeValues" in table_input and not isinstance(table_input["KeyConditionExpression"], str): + del table_input["ExpressionAttributeValues"] + else: + # Pass the original string through. + # The table will accept the string as-is. + pass if "FilterExpression" in table_input: - if table_input["FilterExpression"] in known_query_string_to_condition_map: + if table_input["FilterExpression"] in known_filter_expression_string_to_condition_map: # Turn the query into the resource-formatted query - table_input["FilterExpression"] = known_query_string_to_condition_map[table_input["FilterExpression"]] + table_input["FilterExpression"] = known_filter_expression_string_to_condition_map[table_input["FilterExpression"]] + # boto3 Conditions cannot accept any externally-provided ExpressionAttributeValues + # if the FilterExpression is not a string. + # If the FilterExpression was replaced, remove the now-useless ExpressionAttributeValues. + if "ExpressionAttributeValues" in table_input and not isinstance(table_input["FilterExpression"], str): + del table_input["ExpressionAttributeValues"] + else: + # Pass the original string through. + # The table will accept the string as-is. + pass table_output = self._table.scan(**table_input) client_output = self._resource_shape_to_client_shape_converter.scan_response(table_output) return client_output @@ -171,13 +236,30 @@ def query(self, **kwargs): # convert the string-based KeyConditionExpression and FilterExpression # into the boto3.conditions.Key and boto3.conditions.Attr resource-formatted queries. if "KeyConditionExpression" in table_input: - if table_input["KeyConditionExpression"] in known_query_string_to_condition_map: - # Turn the query into the resource-formatted query - table_input["KeyConditionExpression"] = known_query_string_to_condition_map[table_input["KeyConditionExpression"]] + if table_input["KeyConditionExpression"] in known_key_condition_expression_string_to_condition_map: + table_input["KeyConditionExpression"] = known_key_condition_expression_string_to_condition_map[table_input["KeyConditionExpression"]] + # boto3 Conditions cannot accept any externally-provided ExpressionAttributeValues + # if the KeyConditionExpression is not a string. + # If the KeyConditionExpression was replaced, remove the now-useless ExpressionAttributeValues. + if "ExpressionAttributeValues" in table_input and not isinstance(table_input["KeyConditionExpression"], str): + del table_input["ExpressionAttributeValues"] + else: + # Pass the original string through. + # The table will accept the string as-is. + pass if "FilterExpression" in table_input: - if table_input["FilterExpression"] in known_query_string_to_condition_map: + if table_input["FilterExpression"] in known_filter_expression_string_to_condition_map: # Turn the query into the resource-formatted query - table_input["FilterExpression"] = known_query_string_to_condition_map[table_input["FilterExpression"]] + table_input["FilterExpression"] = known_filter_expression_string_to_condition_map[table_input["FilterExpression"]] + # boto3 Conditions cannot accept any externally-provided ExpressionAttributeValues + # if the FilterExpression is not a string. + # If the FilterExpression was replaced, remove the now-useless ExpressionAttributeValues. + if "ExpressionAttributeValues" in table_input and not isinstance(table_input["FilterExpression"], str): + del table_input["ExpressionAttributeValues"] + else: + # Pass the original string through. + # The table will accept the string as-is. + pass table_output = self._table.query(**table_input) client_output = self._resource_shape_to_client_shape_converter.query_response(table_output) return client_output From 4e312bf15c3f5174b357e1baecf6200c3de6ca34 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 29 May 2025 14:59:21 -0700 Subject: [PATCH 9/9] sync --- .../runtimes/python/test/integ/encrypted/test_table.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py index 48bd3f0da..46436e468 100644 --- a/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py +++ b/DynamoDbEncryption/runtimes/python/test/integ/encrypted/test_table.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 import uuid from copy import deepcopy