From a02c71cdb0042710ed15ed65808ca0c5acf23295 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 May 2025 10:32:02 -0700 Subject: [PATCH 1/9] sync --- .../internal/client_to_resource.py | 138 +++ .../internal/condition_expression_builder.py | 189 +++ .../internal/resource_to_client.py | 167 +++ .../runtimes/python/test/items.py | 61 + .../runtimes/python/test/requests.py | 566 +++++++++ .../runtimes/python/test/responses.py | 167 +++ .../python/test/unit/internal/__init__.py | 2 + .../unit/internal/test_client_to_resource.py | 724 ++++++++++++ .../unit/internal/test_resource_to_client.py | 1024 +++++++++++++++++ 9 files changed, 3038 insertions(+) create mode 100644 DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/client_to_resource.py create mode 100644 DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/condition_expression_builder.py create mode 100644 DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/resource_to_client.py create mode 100644 DynamoDbEncryption/runtimes/python/test/items.py create mode 100644 DynamoDbEncryption/runtimes/python/test/requests.py create mode 100644 DynamoDbEncryption/runtimes/python/test/responses.py create mode 100644 DynamoDbEncryption/runtimes/python/test/unit/internal/__init__.py create mode 100644 DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py create mode 100644 DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/client_to_resource.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/client_to_resource.py new file mode 100644 index 000000000..9665a55c2 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/client_to_resource.py @@ -0,0 +1,138 @@ +from aws_cryptography_internal_dynamodb.smithygenerated.com_amazonaws_dynamodb.boto3_conversions import ( + InternalBoto3DynamoDBFormatConverter, +) +from boto3.dynamodb.types import TypeDeserializer + + +class ClientShapeToResourceShapeConverter: + + def __init__(self, delete_table_name=True): + self.delete_table_name = delete_table_name + self.boto3_converter = InternalBoto3DynamoDBFormatConverter( + item_handler=TypeDeserializer().deserialize, condition_handler=self.condition_handler + ) + + def condition_handler(self, expression_key, request): + """Returns the input condition/names/values as-is.""" + # Conditions do not need to be converted from strings to boto3 Attrs. + # Resources accept either strings or Attrs. + condition = request[expression_key] + + # This conversion in client_to_resource does not update neither + # ExpressionAttributeNames nor ExpressionAttributeValues. + # However, resource_to_client condition_handler may add new + # ExpressionAttributeNames and ExpressionAttributeValues. + # Smithy-generated code expects condition_handlers to return + # ExpressionAttributeNames and ExpressionAttributeValues. + try: + names = request["ExpressionAttributeNames"] + except KeyError: + names = {} + + try: + values = request["ExpressionAttributeValues"] + except KeyError: + values = {} + return condition, names, values + + def put_item_request(self, put_item_request): + out = self.boto3_converter.PutItemInput(put_item_request) + # put_item requests on a boto3.resource.Table do not have a table name. + if self.delete_table_name: + del out["TableName"] + return out + + def put_item_response(self, put_item_response): + return self.boto3_converter.PutItemOutput(put_item_response) + + def get_item_request(self, get_item_request): + out = self.boto3_converter.GetItemInput(get_item_request) + # get_item requests on a boto3.resource.Table do not have a table name. + if self.delete_table_name: + del out["TableName"] + return out + + def get_item_response(self, get_item_response): + return self.boto3_converter.GetItemOutput(get_item_response) + + def query_request(self, query_request): + out = self.boto3_converter.QueryInput(query_request) + # query requests on a boto3.resource.Table do not have a table name. + if self.delete_table_name: + del out["TableName"] + return out + + def query_response(self, query_response): + return self.boto3_converter.QueryOutput(query_response) + + def scan_request(self, scan_request): + out = self.boto3_converter.ScanInput(scan_request) + # scan requests on a boto3.resource.Table do not have a table name. + if self.delete_table_name: + del out["TableName"] + return out + + def delete_item_request(self, delete_item_request): + out = self.boto3_converter.DeleteItemInput(delete_item_request) + # delete_item requests on a boto3.resource.Table do not have a table name. + if self.delete_table_name: + del out["TableName"] + return out + + def update_item_request(self, update_item_request): + out = self.boto3_converter.UpdateItemInput(update_item_request) + # update_item requests on a boto3.resource.Table do not have a table name. + if self.delete_table_name: + del out["TableName"] + return out + + def scan_response(self, scan_response): + return self.boto3_converter.ScanOutput(scan_response) + + def transact_get_items_request(self, transact_get_items_request): + return self.boto3_converter.TransactGetItemsInput(transact_get_items_request) + + def transact_get_items_response(self, transact_get_items_response): + return self.boto3_converter.TransactGetItemsOutput(transact_get_items_response) + + def transact_write_items_request(self, transact_write_items_request): + return self.boto3_converter.TransactWriteItemsInput(transact_write_items_request) + + def transact_write_items_response(self, transact_write_items_response): + return self.boto3_converter.TransactWriteItemsOutput(transact_write_items_response) + + def batch_get_item_request(self, batch_get_item_request): + return self.boto3_converter.BatchGetItemInput(batch_get_item_request) + + def batch_get_item_response(self, batch_get_item_response): + return self.boto3_converter.BatchGetItemOutput(batch_get_item_response) + + def batch_write_item_request(self, batch_write_item_request): + return self.boto3_converter.BatchWriteItemInput(batch_write_item_request) + + def batch_write_item_response(self, batch_write_item_response): + return self.boto3_converter.BatchWriteItemOutput(batch_write_item_response) + + def update_item_response(self, update_item_response): + return self.boto3_converter.UpdateItemOutput(update_item_response) + + def batch_execute_statement_request(self, batch_execute_statement_request): + return self.boto3_converter.BatchExecuteStatementInput(batch_execute_statement_request) + + def batch_execute_statement_response(self, batch_execute_statement_response): + return self.boto3_converter.BatchExecuteStatementOutput(batch_execute_statement_response) + + def delete_item_response(self, delete_item_response): + return self.boto3_converter.DeleteItemOutput(delete_item_response) + + def execute_statement_request(self, execute_statement_request): + return self.boto3_converter.ExecuteStatementInput(execute_statement_request) + + def execute_statement_response(self, execute_statement_response): + return self.boto3_converter.ExecuteStatementOutput(execute_statement_response) + + def execute_transaction_request(self, execute_transaction_request): + return self.boto3_converter.ExecuteTransactionInput(execute_transaction_request) + + def execute_transaction_response(self, execute_transaction_response): + return self.boto3_converter.ExecuteTransactionOutput(execute_transaction_response) diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/condition_expression_builder.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/condition_expression_builder.py new file mode 100644 index 000000000..913d59f78 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/condition_expression_builder.py @@ -0,0 +1,189 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import re + +from boto3.dynamodb.conditions import AttributeBase, BuiltConditionExpression, ConditionBase, Key +from boto3.exceptions import ( + DynamoDBNeedsConditionError, + DynamoDBNeedsKeyConditionError, +) + +ATTR_NAME_REGEX = re.compile(r"[^.\[\]]+(?![^\[]*\])") + + +class InternalDBESDKDynamoDBConditionExpressionBuilder: + """This class is used to build condition expressions with placeholders""" + + def __init__(self): + self._name_count = 0 + self._value_count = 0 + self._name_placeholder = "n" + self._value_placeholder = "v" + + def _get_name_placeholder(self): + return "#" + self._name_placeholder + str(self._name_count) + + def _get_value_placeholder(self): + return ":" + self._value_placeholder + str(self._value_count) + + def reset(self): + """Resets the placeholder name and values""" + self._name_count = 0 + self._value_count = 0 + + def build_expression( + self, condition, attribute_name_placeholders, attribute_value_placeholders, is_key_condition=False + ): + """ + Builds the condition expression and the dictionary of placeholders. + + :type condition: ConditionBase + :param condition: A condition to be built into a condition expression + string with any necessary placeholders. + + :type is_key_condition: Boolean + :param is_key_condition: True if the expression is for a + KeyConditionExpression. False otherwise. + + :rtype: (string, dict, dict) + :returns: Will return a string representing the condition with + placeholders inserted where necessary, a dictionary of + placeholders for attribute names, and a dictionary of + placeholders for attribute values. Here is a sample return value: + + ('#n0 = :v0', {'#n0': 'myattribute'}, {':v1': 'myvalue'}) + """ + if not isinstance(condition, ConditionBase): + raise DynamoDBNeedsConditionError(condition) + condition_expression = self._build_expression( + condition, + attribute_name_placeholders, + attribute_value_placeholders, + is_key_condition=is_key_condition, + ) + print(f"BuiltConditionExpression {condition_expression=}") + return BuiltConditionExpression( + condition_expression=condition_expression, + attribute_name_placeholders=attribute_name_placeholders, + attribute_value_placeholders=attribute_value_placeholders, + ) + + def _build_expression( + self, + condition, + attribute_name_placeholders, + attribute_value_placeholders, + is_key_condition, + ): + expression_dict = condition.get_expression() + replaced_values = [] + for value in expression_dict["values"]: + # Build the necessary placeholders for that value. + # Placeholders are built for both attribute names and values. + replaced_value = self._build_expression_component( + value, + attribute_name_placeholders, + attribute_value_placeholders, + condition.has_grouped_values, + is_key_condition, + ) + replaced_values.append(replaced_value) + # Fill out the expression using the operator and the + # values that have been replaced with placeholders. + return expression_dict["format"].format(*replaced_values, operator=expression_dict["operator"]) + + def _build_expression_component( + self, + value, + attribute_name_placeholders, + attribute_value_placeholders, + has_grouped_values, + is_key_condition, + ): + # Continue to recurse if the value is a ConditionBase in order + # to extract out all parts of the expression. + if isinstance(value, ConditionBase): + return self._build_expression( + value, + attribute_name_placeholders, + attribute_value_placeholders, + is_key_condition, + ) + # If it is not a ConditionBase, we can recurse no further. + # So we check if it is an attribute and add placeholders for + # its name + elif isinstance(value, AttributeBase): + if is_key_condition and not isinstance(value, Key): + raise DynamoDBNeedsKeyConditionError( + f"Attribute object {value.name} is of type {type(value)}. " + f"KeyConditionExpression only supports Attribute objects " + f"of type Key" + ) + return self._build_name_placeholder(value, attribute_name_placeholders) + # If it is anything else, we treat it as a value and thus placeholders + # are needed for the value. + else: + return self._build_value_placeholder(value, attribute_value_placeholders, has_grouped_values) + + def _build_name_placeholder(self, value, attribute_name_placeholders): + attribute_name = value.name + # Figure out which parts of the attribute name that needs replacement. + attribute_name_parts = ATTR_NAME_REGEX.findall(attribute_name) + + # Add a temporary placeholder for each of these parts. + placeholder_format = ATTR_NAME_REGEX.sub("%s", attribute_name) + str_format_args = [] + for part in attribute_name_parts: + # If the the name is already an AttributeName, use it. Don't make a new placeholder. + if part in attribute_name_placeholders: + str_format_args.append(part) + else: + name_placeholder = self._get_name_placeholder() + self._name_count += 1 + str_format_args.append(name_placeholder) + # Add the placeholder and value to dictionary of name placeholders. + attribute_name_placeholders[name_placeholder] = part + # Replace the temporary placeholders with the designated placeholders. + return placeholder_format % tuple(str_format_args) + + def _build_value_placeholder(self, value, attribute_value_placeholders, has_grouped_values=False): + print(f"{attribute_value_placeholders=}") + # If the values are grouped, we need to add a placeholder for + # each element inside of the actual value. + + # Also, you can define a grouped value with a colon here. + # If it's a colon, it's not a grouped value for the sake of this logic. + # Treat it as an "else" case. + if has_grouped_values: + placeholder_list = [] + # If it's a pre-defined grouped attribute, don't attempt to unpack it as if it were + for v in value: + print(f"v1 {v=}") + # If the value is already an AttributeValue, reuse it. Don't make a new placeholder. + if v in attribute_value_placeholders: + print("in") + placeholder_list.append(v) + else: + print("not in") + value_placeholder = self._get_value_placeholder() + self._value_count += 1 + placeholder_list.append(value_placeholder) + attribute_value_placeholders[value_placeholder] = v + # Assuming the values are grouped by parenthesis. + # IN is the currently the only one that uses this so it maybe + # needed to be changed in future. + return "(" + ", ".join(placeholder_list) + ")" + # Otherwise, treat the value as a single value that needs only + # one placeholder. + else: + print(f"v2 {value=}") + # If the value is already an AttributeValue, reuse it. Don't make a new placeholder. + if value in attribute_value_placeholders: + print("in") + return value + else: + print("not in") + value_placeholder = self._get_value_placeholder() + self._value_count += 1 + attribute_value_placeholders[value_placeholder] = value + return value_placeholder diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/resource_to_client.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/resource_to_client.py new file mode 100644 index 000000000..93afbd9a0 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/resource_to_client.py @@ -0,0 +1,167 @@ +from aws_cryptography_internal_dynamodb.smithygenerated.com_amazonaws_dynamodb.boto3_conversions import ( + InternalBoto3DynamoDBFormatConverter, +) +from boto3.dynamodb.types import TypeSerializer + +from aws_dbesdk_dynamodb.internal.condition_expression_builder import InternalDBESDKDynamoDBConditionExpressionBuilder + + +class ResourceShapeToClientShapeConverter: + + def __init__(self, table_name=None): + self.boto3_converter = InternalBoto3DynamoDBFormatConverter( + item_handler=TypeSerializer().serialize, condition_handler=self.condition_handler + ) + self.table_name = table_name + self.expression_builder = InternalDBESDKDynamoDBConditionExpressionBuilder() + + def condition_handler(self, expression_key, request): + """ + Converts an object from boto3.dynamodb.conditions to a string + and updates ExpressionAttributeNames and ExpressionAttributeValues with any new names/values. + The ExpressionAttributeValues are returned in resource format (Python dictionaries). + """ + condition_expression = request[expression_key] + + try: + existing_expression_attribute_names = request["ExpressionAttributeNames"] + except KeyError: + existing_expression_attribute_names = {} + try: + existing_expression_attribute_values = request["ExpressionAttributeValues"] + except KeyError: + existing_expression_attribute_values = {} + + # Only convert if the condition expression is a boto3.dynamodb.conditions object. + # Resources also accept strings. + # If condition is not from boto3.dynamodb.conditions, assume the condition is string-like, and return as-is. + if ( + hasattr(condition_expression, "__module__") + and condition_expression.__module__ == "boto3.dynamodb.conditions" + ): + built_condition_expression = self.expression_builder.build_expression( + condition_expression, existing_expression_attribute_names, existing_expression_attribute_values + ) + else: + return condition_expression, existing_expression_attribute_names, existing_expression_attribute_values + + # Unpack returned BuiltConditionExpression. + expression_str = built_condition_expression.condition_expression + attribute_names_from_built_expression = built_condition_expression.attribute_name_placeholders + # Join any placeholder ExpressionAttributeNames with any other ExpressionAttributeNames. + # The BuiltConditionExpression will return new names, not any that already exist. + # The two sets of names must be joined to form the complete set of names for the condition expression. + try: + out_names = request["ExpressionAttributeNames"] | attribute_names_from_built_expression + except KeyError: + out_names = attribute_names_from_built_expression + # Join existing and new values. + attribute_values_from_built_expression = built_condition_expression.attribute_value_placeholders + try: + out_values = request["ExpressionAttributeValues"] | attribute_values_from_built_expression + except KeyError: + out_values = attribute_values_from_built_expression + + return expression_str, out_names, out_values + + def put_item_request(self, put_item_request): + # put_item requests on a boto3.resource.Table require a configured table name. + if not self.table_name: + raise ValueError("Table name must be provided to ResourceShapeToClientShapeConverter to use put_item") + put_item_request["TableName"] = self.table_name + return self.boto3_converter.PutItemInput(put_item_request) + + def get_item_request(self, get_item_request): + # get_item requests on a boto3.resource.Table require a configured table name. + if not self.table_name: + raise ValueError("Table name must be provided to ResourceShapeToClientShapeConverter to use get_item") + get_item_request["TableName"] = self.table_name + return self.boto3_converter.GetItemInput(get_item_request) + + def query_request(self, query_request): + # query requests on a boto3.resource.Table require a configured table name. + if not self.table_name: + raise ValueError("Table name must be provided to ResourceShapeToClientShapeConverter to use query") + query_request["TableName"] = self.table_name + return self.boto3_converter.QueryInput(query_request) + + def scan_request(self, scan_request): + # scan requests on a boto3.resource.Table require a configured table name. + if not self.table_name: + raise ValueError("Table name must be provided to ResourceShapeToClientShapeConverter to use scan") + scan_request["TableName"] = self.table_name + return self.boto3_converter.ScanInput(scan_request) + + def update_item_request(self, update_item_request): + # update_item requests on a boto3.resource.Table require a configured table name. + if not self.table_name: + raise ValueError("Table name must be provided to ResourceShapeToClientShapeConverter to use update_item") + update_item_request["TableName"] = self.table_name + return self.boto3_converter.UpdateItemInput(update_item_request) + + def delete_item_request(self, delete_item_request): + # delete_item requests on a boto3.resource.Table require a configured table name. + if not self.table_name: + raise ValueError("Table name must be provided to ResourceShapeToClientShapeConverter to use delete_item") + delete_item_request["TableName"] = self.table_name + return self.boto3_converter.DeleteItemInput(delete_item_request) + + def transact_get_items_request(self, transact_get_items_request): + return self.boto3_converter.TransactGetItemsInput(transact_get_items_request) + + def transact_get_items_response(self, transact_get_items_response): + return self.boto3_converter.TransactGetItemsOutput(transact_get_items_response) + + def transact_write_items_request(self, transact_write_items_request): + return self.boto3_converter.TransactWriteItemsInput(transact_write_items_request) + + def transact_write_items_response(self, transact_write_items_response): + return self.boto3_converter.TransactWriteItemsOutput(transact_write_items_response) + + def batch_get_item_request(self, batch_get_item_request): + return self.boto3_converter.BatchGetItemInput(batch_get_item_request) + + def batch_get_item_response(self, batch_get_item_response): + return self.boto3_converter.BatchGetItemOutput(batch_get_item_response) + + def batch_write_item_request(self, batch_write_item_request): + return self.boto3_converter.BatchWriteItemInput(batch_write_item_request) + + def batch_write_item_response(self, batch_write_item_response): + return self.boto3_converter.BatchWriteItemOutput(batch_write_item_response) + + def batch_execute_statement_request(self, batch_execute_statement_request): + return self.boto3_converter.BatchExecuteStatementInput(batch_execute_statement_request) + + def batch_execute_statement_response(self, batch_execute_statement_response): + return self.boto3_converter.BatchExecuteStatementOutput(batch_execute_statement_response) + + def execute_statement_request(self, execute_statement_request): + return self.boto3_converter.ExecuteStatementInput(execute_statement_request) + + def execute_statement_response(self, execute_statement_response): + return self.boto3_converter.ExecuteStatementOutput(execute_statement_response) + + def execute_transaction_request(self, execute_transaction_request): + return self.boto3_converter.ExecuteTransactionInput(execute_transaction_request) + + def execute_transaction_response(self, execute_transaction_response): + return self.boto3_converter.ExecuteTransactionOutput(execute_transaction_response) + + def scan_response(self, scan_response): + return self.boto3_converter.ScanOutput(scan_response) + + def query_response(self, query_response): + return self.boto3_converter.QueryOutput(query_response) + + def get_item_response(self, get_item_response): + return self.boto3_converter.GetItemOutput(get_item_response) + + def put_item_response(self, put_item_response): + return self.boto3_converter.PutItemOutput(put_item_response) + + def update_item_response(self, update_item_response): + return self.boto3_converter.UpdateItemOutput(update_item_response) + + def delete_item_response(self, delete_item_response): + return self.boto3_converter.DeleteItemOutput(delete_item_response) diff --git a/DynamoDbEncryption/runtimes/python/test/items.py b/DynamoDbEncryption/runtimes/python/test/items.py new file mode 100644 index 000000000..fcdbf4278 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/test/items.py @@ -0,0 +1,61 @@ +from decimal import Decimal + +simple_item_ddb = { + "partition_key": {"S": "test-key"}, + "sort_key": {"N": "1"}, + "attribute1": {"S": "encrypted value"}, + "attribute2": {"S": "signed value"}, + ":attribute3": {"S": "unsigned value"}, +} + +simple_key_ddb = {"partition_key": simple_item_ddb["partition_key"], "sort_key": simple_item_ddb["sort_key"]} + +simple_item_dict = { + "partition_key": "test-key", + "sort_key": 1, + "attribute1": "encrypted value", + "attribute2": "signed value", + ":attribute3": "unsigned value", +} + +simple_key_dict = {"partition_key": simple_item_dict["partition_key"], "sort_key": simple_item_dict["sort_key"]} + +complex_item_ddb = { + "partition_key": {"S": "all-types-test"}, + "sort_key": {"N": "1"}, + "attribute1": { + "M": { + "string": {"S": "string value"}, + "number": {"N": "123.45"}, + "binary": {"B": b"binary data"}, + "string_set": {"SS": ["value1", "value2"]}, + "number_set": {"NS": ["1", "2", "3"]}, + "binary_set": {"BS": [b"binary1", b"binary2"]}, + "list": {"L": [{"S": "list item 1"}, {"N": "42"}, {"B": b"list binary"}]}, + "map": {"M": {"nested_string": {"S": "nested value"}, "nested_number": {"N": "42"}}}, + } + }, + "attribute2": {"S": "signed value"}, + ":attribute3": {"S": "unsigned value"}, +} + +complex_key_ddb = {"partition_key": complex_item_ddb["partition_key"], "sort_key": complex_item_ddb["sort_key"]} + +complex_item_dict = { + "partition_key": "all-types-test", + "sort_key": 1, + "attribute1": { + "string": "string value", + "number": Decimal("123.45"), + "binary": b"binary data", + "string_set": {"value1", "value2"}, + "number_set": {Decimal("1"), 2, Decimal("3")}, + "binary_set": {b"binary1", b"binary2"}, + "list": ["list item 1", 42, b"list binary"], + "map": {"nested_string": "nested value", "nested_number": 42}, + }, + "attribute2": "signed value", + ":attribute3": "unsigned value", +} + +complex_key_dict = {"partition_key": complex_item_dict["partition_key"], "sort_key": complex_item_dict["sort_key"]} diff --git a/DynamoDbEncryption/runtimes/python/test/requests.py b/DynamoDbEncryption/runtimes/python/test/requests.py new file mode 100644 index 000000000..544af3ce8 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/test/requests.py @@ -0,0 +1,566 @@ +"""Request constants for DynamoDB operations used for testing.""" + +from boto3.dynamodb.conditions import Attr, Key + +from .constants import ( + INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, + INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME_PLAINTEXT, +) + +# Base request structures that are shared between DDB and dict formats + + +def base_put_item_request(item): + """Base structure for put_item requests.""" + return {"Item": item} + + +def base_get_item_request(item): + """Base structure for get_item requests.""" + return {"Key": {"partition_key": item["partition_key"], "sort_key": item["sort_key"]}} + + +def base_delete_item_request(item): + """Base structure for delete_item requests.""" + return {"Key": {"partition_key": item["partition_key"], "sort_key": item["sort_key"]}} + + +def base_query_request(item): + """Base structure for query requests.""" + return { + "KeyConditionExpression": "partition_key = :pk", + "ExpressionAttributeValues": {":pk": item["partition_key"]}, + } + + +def base_scan_request(item): + """Base structure for scan requests.""" + return { + "FilterExpression": "attribute2 = :a2", + "ExpressionAttributeValues": {":a2": item["attribute2"]}, + } + + +def base_batch_write_item_request(actions_with_items): + """Base structure for batch_write_item requests.""" + return {"RequestItems": {INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME: actions_with_items}} + + +def base_batch_get_item_request(keys): + """Base structure for batch_get_item requests.""" + return {"RequestItems": {INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME: {"Keys": keys}}} + + +def base_transact_write_item_request(actions_with_items): + """Base structure for transact_write_item requests.""" + return {"TransactItems": actions_with_items} + + +def base_transact_get_item_request(keys): + """Base structure for transact_get_item requests.""" + return { + "TransactItems": [{"Get": {"TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, "Key": key}} for key in keys] + } + + +def base_update_item_request_signed_attribute(item): + """Base structure for update_item requests.""" + return { + "Key": {"partition_key": item["partition_key"], "sort_key": item["sort_key"]}, + "UpdateExpression": "SET attribute1 = :val", + "ExpressionAttributeValues": {":val": item["attribute1"]}, + } + + +def base_update_item_request_unsigned_attribute(item): + """Base structure for update_item requests.""" + return { + "Key": {"partition_key": item["partition_key"], "sort_key": item["sort_key"]}, + "UpdateExpression": "SET #attr3 = :val", + "ExpressionAttributeValues": {":val": item[":attribute3"]}, + "ExpressionAttributeNames": {"#attr3": ":attribute3"}, + } + + +def basic_execute_statement_request_encrypted_table(item): + """Base structure for execute_statement requests for an encrypted table.""" + return { + "Statement": f"""SELECT * FROM {INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME} + WHERE partition_key=? AND sort_key=?""", + "Parameters": [item["partition_key"], item["sort_key"]], + } + + +def basic_execute_statement_request_plaintext_table(item): + """Base structure for execute_statement requests for a plaintext table.""" + return { + "Statement": f"""SELECT * FROM {INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME_PLAINTEXT} + WHERE partition_key=? AND sort_key=?""", + "Parameters": [item["partition_key"], item["sort_key"]], + } + + +def basic_execute_transaction_request_encrypted_table(item): + """Base structure for execute_transaction requests for an encrypted table.""" + return { + "TransactStatements": [ + { + "Statement": f"""SELECT * FROM {INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME} + WHERE partition_key=? AND sort_key=?""", + "Parameters": [item["partition_key"], item["sort_key"]], + } + ] + } + + +def basic_execute_transaction_request_plaintext_table(item): + """Base structure for execute_transaction requests for a plaintext table.""" + return { + "TransactStatements": [ + { + "Statement": f"""SELECT * FROM {INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME_PLAINTEXT} + WHERE partition_key=? AND sort_key=?""", + "Parameters": [item["partition_key"], item["sort_key"]], + } + ] + } + + +def basic_batch_execute_statement_request_encrypted_table(): + """Base structure for batch_execute_statement requests.""" + return {"Statements": [{"Statement": "SELECT * FROM " + INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME}]} + + +def basic_batch_execute_statement_request_plaintext_table(): + """Base structure for batch_execute_statement requests for a plaintext table.""" + return {"Statements": [{"Statement": "SELECT * FROM " + INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME_PLAINTEXT}]} + + +# Base exhaustive request structures that are shared between DDB and dict formats + +# No exhaustive requests are intended to be able to be used as real requests. +# Some parameters conflict with each other when sent to DynamoDB. +# These are only intended to test the conversion of the request from client to resource format. + + +def base_exhaustive_put_item_request(item): + """ + Base structure for exhaustive put_item requests. + This is not intended to be able to be used as a real request. + Some parameters conflict with each other when sent to DynamoDB. + This is only intended to test the conversion of the request from client to resource format. + """ + return { + # Expected is legacy, but still in the boto3 docs. + "Expected": { + "partition_key": { + "Value": item["partition_key"], + }, + "sort_key": {"AttributeValueList": [item["sort_key"]], "ComparisonOperator": "EQ"}, + }, + "ExpressionAttributeNames": {"#pk": "partition_key", "#sk": "sort_key"}, + "ExpressionAttributeValues": {":pk": item["partition_key"], ":sk": item["sort_key"]}, + "ReturnConsumedCapacity": "TOTAL", + "ReturnItemCollectionMetrics": "SIZE", + "ReturnValues": "ALL_OLD", + "ReturnValuesOnConditionCheckFailure": "ALL_OLD", + } + + +def base_exhaustive_get_item_request(item): + """ + Base structure for exhaustive get_item requests. + This is not intended to be able to be used as a real request. + Some parameters conflict with each other when sent to DynamoDB. + This is only intended to test the conversion of the request from client to resource format. + """ + return { + "ReturnConsumedCapacity": "TOTAL", + "ReturnItemCollectionMetrics": "SIZE", + "ProjectionExpression": "partition_key, sort_key, attribute1, attribute2", + "ExpressionAttributeNames": { + "#pk": "partition_key", + "#sk": "sort_key", + "#a1": "attribute1", + "#a2": "attribute2", + }, + "ConsistentRead": True, + "AttributesToGet": ["partition_key", "sort_key", "attribute1", "attribute2"], + } + + +def base_exhaustive_delete_item_request(item): + """ + Base structure for exhaustive delete_item requests. + This is not intended to be able to be used as a real request. + Some parameters conflict with each other when sent to DynamoDB. + This is only intended to test the conversion of the request from client to resource format. + """ + return { + "ReturnConsumedCapacity": "TOTAL", + "ReturnItemCollectionMetrics": "SIZE", + "ReturnValues": "ALL_OLD", + "ReturnValuesOnConditionCheckFailure": "ALL_OLD", + } + + +def base_exhaustive_query_request(item): + """ + Base structure for exhaustive query requests. + This is not intended to be able to be used as a real request. + Some parameters conflict with each other when sent to DynamoDB. + This is only intended to test the conversion of the request from client to resource format. + """ + return { + "IndexName": "index_name", + "Select": "SPECIFIC_ATTRIBUTES", + "AttributesToGet": ["partition_key", "sort_key", "attribute1", "attribute2"], + "KeyConditions": {"partition_key": {"AttributeValueList": [item["partition_key"]], "ComparisonOperator": "EQ"}}, + "QueryFilter": {"attribute1": {"AttributeValueList": [item["attribute1"]], "ComparisonOperator": "EQ"}}, + "ConditionalOperator": "AND", + "ScanIndexForward": True, + "ExclusiveStartKey": {"partition_key": item["partition_key"], "sort_key": item["sort_key"]}, + "ReturnConsumedCapacity": "TOTAL", + "ProjectionExpression": "partition_key, sort_key, attribute1, attribute2", + "FilterExpression": "attribute1 = :a1", + "ExpressionAttributeNames": { + "#pk": "partition_key", + "#sk": "sort_key", + "#a1": "attribute1", + "#a2": "attribute2", + }, + "ExpressionAttributeValues": {":pk": item["partition_key"], ":a1": item["attribute1"]}, + } + + +def base_exhaustive_scan_request(item): + """ + Base structure for exhaustive scan requests. + This is not intended to be able to be used as a real request. + Some parameters conflict with each other when sent to DynamoDB. + This is only intended to test the conversion of the request from client to resource format. + """ + return { + "IndexName": "index_name", + "AttributesToGet": ["partition_key", "sort_key", "attribute1", "attribute2"], + "Select": "SPECIFIC_ATTRIBUTES", + "ScanFilter": {"attribute1": {"AttributeValueList": [item["attribute1"]], "ComparisonOperator": "EQ"}}, + "ConditionalOperator": "AND", + "ReturnConsumedCapacity": "TOTAL", + "ReturnItemCollectionMetrics": "SIZE", + "ExpressionAttributeNames": {"#a1": "attribute1"}, + "ExpressionAttributeValues": {":a1": item["attribute1"]}, + "ExclusiveStartKey": {"partition_key": item["partition_key"], "sort_key": item["sort_key"]}, + } + + +# DDB format request functions + + +def basic_put_item_request_ddb(item): + """Get a put_item request in DDB format for any item.""" + base = base_put_item_request(item) + return {"TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, **base} + + +def exhaustive_put_item_request_ddb(item): + """Get a put_item request in DDB format for any item.""" + base = basic_put_item_request_ddb(item) + additional_keys = base_exhaustive_put_item_request(item) + additional_keys["ConditionExpression"] = "attribute_not_exists(#pk) AND attribute_not_exists(#sk)" + return {**base, **additional_keys} + + +def basic_get_item_request_ddb(item): + """Get a get_item request in DDB format for any item.""" + base = base_get_item_request(item) + return {"TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, **base} + + +def exhaustive_get_item_request_ddb(item): + """Get a get_item request in DDB format for any item.""" + base = basic_get_item_request_ddb(item) + additional_keys = base_exhaustive_get_item_request(item) + return {**base, **additional_keys} + + +def basic_delete_item_request_ddb(item): + """Get a delete_item request in DDB format for any item.""" + base = base_delete_item_request(item) + return {"TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, **base} + + +def exhaustive_delete_item_request_ddb(item): + """Get a delete_item request in DDB format for any item.""" + base = basic_delete_item_request_ddb(item) + additional_keys = base_exhaustive_delete_item_request(item) + return {**base, **additional_keys} + + +def basic_query_request_ddb(item): + """Get a query request in DDB format for any item.""" + base = base_query_request(item) + return {"TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, **base} + + +def exhaustive_query_request_ddb(item): + """ + Query request with all possible parameters. + This is not intended to be able to be used as a real request. + Some parameters conflict with each other when sent to DynamoDB. + This is only intended to test the conversion of the request from client to resource format. + """ + base = basic_query_request_ddb(item) + additional_keys = base_exhaustive_query_request(item) + return {**base, **additional_keys} + + +def basic_scan_request_ddb(item): + """Get a scan request in DDB format for any item.""" + base = base_scan_request(item) + return {"TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, **base} + + +def exhaustive_scan_request_ddb(item): + """Get a scan request in DDB format for any item.""" + base = basic_scan_request_ddb(item) + additional_keys = base_exhaustive_scan_request(item) + return {**base, **additional_keys} + + +def basic_batch_write_item_request_ddb(actions_with_items): + """Get a batch_write_item request in DDB format for any items.""" + return base_batch_write_item_request(actions_with_items) + + +def basic_batch_write_item_put_request_ddb(items): + """Get a batch_write_item put request in DDB format for any items.""" + actions_with_items = [{"PutRequest": {"Item": item}} for item in items] + return basic_batch_write_item_request_ddb(actions_with_items) + + +def basic_batch_write_item_delete_request_ddb(keys): + """Get a batch_write_item delete request in DDB format for any keys.""" + actions_with_keys = [{"DeleteRequest": {"Key": key}} for key in keys] + return basic_batch_write_item_request_ddb(actions_with_keys) + + +def basic_batch_get_item_request_ddb(keys): + """Get a batch_get_item request in DDB format for any keys.""" + return base_batch_get_item_request(keys) + + +def basic_transact_write_item_request_ddb(actions_with_items): + """Get a transact_write_item request in DDB format for any items.""" + return base_transact_write_item_request(actions_with_items) + + +def basic_transact_write_item_put_request_ddb(items): + """Get a transact_write_item put request in DDB format for any items.""" + actions_with_items = [ + {"Put": {"TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, "Item": item}} for item in items + ] + return basic_transact_write_item_request_ddb(actions_with_items) + + +def basic_transact_write_item_delete_request_ddb(keys): + """Get a transact_write_item delete request in DDB format for any keys.""" + actions_with_keys = [{"Delete": {"TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, "Key": key}} for key in keys] + return basic_transact_write_item_request_ddb(actions_with_keys) + + +def basic_transact_write_item_condition_check_request_ddb(keys): + """Get a transact_write_item condition check request in DDB format for any keys.""" + actions_with_keys = [ + {"ConditionCheck": {"TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, "Key": key}} for key in keys + ] + return basic_transact_write_item_request_ddb(actions_with_keys) + + +def basic_transact_get_item_request_ddb(keys): + """Get a transact_get_item request in DDB format for any keys.""" + return base_transact_get_item_request(keys) + + +def basic_query_paginator_request(key): + """Get a query paginator request in DDB format for any item.""" + return { + "TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, + "KeyConditionExpression": "partition_key = :pk AND sort_key = :sk", + "ExpressionAttributeValues": {":pk": key["partition_key"], ":sk": key["sort_key"]}, + } + + +def basic_scan_paginator_request(item): + """Get a scan paginator request in DDB format for any item.""" + return { + "TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, + "FilterExpression": "partition_key = :pk AND sort_key = :sk", + "ExpressionAttributeValues": {":pk": item["partition_key"], ":sk": item["sort_key"]}, + } + + +def basic_update_item_request_ddb_signed_attribute(item): + """Get an update_item request in DDB format for any item.""" + base = base_update_item_request_signed_attribute(item) + return {"TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, **base} + + +def basic_update_item_request_ddb_unsigned_attribute(item): + """Get an update_item request in DDB format for any item.""" + base = base_update_item_request_unsigned_attribute(item) + return {"TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, **base} + + +# Dict format request functions + + +def basic_put_item_request_dict(item): + """Get a put_item request in dict format for any item.""" + return base_put_item_request(item) + + +def exhaustive_put_item_request_dict(item): + """ + Get a put_item request in dict format for any item. + This is not intended to be able to be used as a real request. + Some parameters conflict with each other when sent to DynamoDB. + This is only intended to test the conversion of the request from client to resource format. + """ + base = basic_put_item_request_dict(item) + # Replace the default ConditionExpression string with a ConditionExpression object + # to increase test coverage. + additional_keys = base_exhaustive_put_item_request(item) + additional_keys["ConditionExpression"] = Attr("#pk").not_exists() & Attr("#sk").not_exists() + return {**base, **additional_keys} + + +def basic_get_item_request_dict(item): + """Get a get_item request in dict format for any item.""" + return base_get_item_request(item) + + +def basic_delete_item_request_dict(item): + """Get a delete_item request in dict format for any item.""" + return base_delete_item_request(item) + + +def exhaustive_get_item_request_dict(item): + """ + Get a get_item request in dict format for any item. + This is not intended to be able to be used as a real request. + Some parameters conflict with each other when sent to DynamoDB. + This is only intended to test the conversion of the request from client to resource format. + """ + base = basic_get_item_request_dict(item) + additional_keys = base_exhaustive_get_item_request(item) + return {**base, **additional_keys} + + +def basic_query_request_dict(item): + """Get a query request in dict format for any item.""" + base = base_query_request(item) + return base + + +def basic_query_request_dict_condition_expression(item): + """Get a query request in dict format for any item.""" + base = base_query_request(item) + # Replace the default KeyConditionExpression string with a ConditionExpression object + # to increase test coverage. + return {"KeyConditionExpression": Key("partition_key").eq(item["partition_key"]), **base} + + +def exhaustive_query_request_dict(item): + """ + Get a query request in dict format for any item. + This is not intended to be able to be used as a real request. + Some parameters conflict with each other when sent to DynamoDB. + This is only intended to test the conversion of the request from client to resource format. + """ + base = basic_query_request_dict(item) + additional_keys = base_exhaustive_query_request(item) + return {**base, **additional_keys} + + +def basic_scan_request_dict(item): + """Get a scan request in dict format for any item.""" + return base_scan_request(item) + + +def exhaustive_scan_request_dict(item): + """ + Get a scan request in dict format for any item. + This is not intended to be able to be used as a real request. + Some parameters conflict with each other when sent to DynamoDB. + This is only intended to test the conversion of the request from client to resource format. + """ + base = basic_scan_request_dict(item) + additional_keys = base_exhaustive_scan_request(item) + return {**base, **additional_keys} + + +def basic_batch_write_item_request_dict(actions_with_items): + """Get a batch_write_item request in dict format for any items.""" + return base_batch_write_item_request(actions_with_items) + + +def basic_batch_write_item_put_request_dict(items): + """Get a batch_put_item request in dict format for any items.""" + actions_with_items = [{"PutRequest": {"Item": item}} for item in items] + return basic_batch_write_item_request_dict(actions_with_items) + + +def basic_batch_write_item_delete_request_dict(keys): + """Get a batch_write_item delete request in dict format for any keys.""" + actions_with_keys = [{"DeleteRequest": {"Key": key}} for key in keys] + return basic_batch_write_item_request_dict(actions_with_keys) + + +def basic_batch_get_item_request_dict(keys): + """Get a batch_get_item request in dict format for any keys.""" + return base_batch_get_item_request(keys) + + +def basic_transact_write_item_request_dict(actions_with_items): + """Get a transact_write_item request in dict format for any items.""" + return base_transact_write_item_request(actions_with_items) + + +def basic_transact_write_item_put_request_dict(items): + """Get a transact_write_item put request in dict format for any items.""" + actions_with_items = [ + {"Put": {"TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, "Item": item}} for item in items + ] + return basic_transact_write_item_request_dict(actions_with_items) + + +def basic_transact_write_item_delete_request_dict(keys): + """Get a transact_write_item delete request in dict format for any keys.""" + actions_with_keys = [{"Delete": {"TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, "Key": key}} for key in keys] + return basic_transact_write_item_request_dict(actions_with_keys) + + +def basic_transact_write_item_condition_check_request_dict(keys): + """Get a transact_write_item condition check request in dict format for any keys.""" + actions_with_keys = [ + {"ConditionCheck": {"TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, "Key": key}} for key in keys + ] + return basic_transact_write_item_request_dict(actions_with_keys) + + +def basic_transact_get_item_request_dict(keys): + """Get a transact_get_item request in dict format for any keys.""" + return base_transact_get_item_request(keys) + + +def basic_update_item_request_dict_signed_attribute(item): + """Get an update_item request in dict format for any item.""" + base = base_update_item_request_signed_attribute(item) + return base + + +def basic_update_item_request_dict_unsigned_attribute(item): + """Get an update_item request in dict format for any item.""" + base = base_update_item_request_unsigned_attribute(item) + return base diff --git a/DynamoDbEncryption/runtimes/python/test/responses.py b/DynamoDbEncryption/runtimes/python/test/responses.py new file mode 100644 index 000000000..6237385d2 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/test/responses.py @@ -0,0 +1,167 @@ +from test.integ.encrypted.test_resource import INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME + + +def basic_put_item_response(item): + """Get a put_item response in resource (ddb) format for any item.""" + return {"Attributes": item} + + +def exhaustive_put_item_response(item): + """ + Get a put_item response in resource (ddb) format for any item. + This is not intended to be a real response that DynamoDB would return, + but the response should contain additional attributes that DynamoDB could return. + This is only intended to exhaustively test the conversion of the request from client to resource format. + """ + base = basic_put_item_response(item) + additional_keys = { + "ConsumedCapacity": {"CapacityUnits": 1, "TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME}, + "ItemCollectionMetrics": { + "TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, + "ItemCollectionKey": {"partition_key": item["partition_key"]}, + }, + "SequenceNumber": "1234567890", + "SizeEstimateRangeGB": [0.5, 1.0], + } + return {**base, **additional_keys} + + +def basic_get_item_response(item): + """Get a get_item response in resource (ddb) format for any item.""" + return {"Item": item} + + +def exhaustive_get_item_response(item): + """ + Get a get_item response in resource (ddb) format for any item. + This is not intended to be a real response that DynamoDB would return, + but the response should contain additional attributes that DynamoDB could return. + This is only intended to exhaustively test the conversion of the request from client to resource format. + """ + base = basic_get_item_response(item) + additional_keys = { + "ConsumedCapacity": {"CapacityUnits": 1, "TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME}, + } + return {**base, **additional_keys} + + +def basic_query_response(items): + """Get a query response in resource (ddb) format for any items.""" + return { + "Items": items, + "Count": len(items), + "ScannedCount": len(items), + "ConsumedCapacity": {"CapacityUnits": 1, "TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME}, + } + + +def exhaustive_query_response(items): + """ + Get a query response in resource (ddb) format for any items. + This is not intended to be a real response that DynamoDB would return, + but the response should contain additional attributes that DynamoDB could return. + This is only intended to exhaustively test the conversion of the request from client to resource format. + """ + base = basic_query_response(items) + additional_keys = { + "LastEvaluatedKey": {"partition_key": items[-1]["partition_key"]}, + } + return {**base, **additional_keys} + + +def basic_scan_response(items, keys): + """Get a scan response in resource (ddb) format for any items.""" + return { + "Items": items, + } + + +def exhaustive_scan_response(items, keys): + """ + Get a scan response in resource (ddb) format for any items. + This is not intended to be a real response that DynamoDB would return, + but the response should contain additional attributes that DynamoDB could return. + This is only intended to exhaustively test the conversion of the request from client to resource format. + """ + base = basic_scan_response(items, keys) + additional_keys = { + "ConsumedCapacity": {"CapacityUnits": 1, "TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME}, + "Count": len(items), + "ScannedCount": len(items), + "LastEvaluatedKey": keys[-1], + } + return {**base, **additional_keys} + + +def basic_batch_get_item_response(items): + """Get a batch_get_item response in resource (ddb) format for any items.""" + return {"Responses": {INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME: items}} + + +def exhaustive_batch_get_item_response(items): + """ + Get a batch_get_item response in resource (ddb) format for any items. + This is not intended to be a real response that DynamoDB would return, + but the response should contain additional attributes that DynamoDB could return. + This is only intended to exhaustively test the conversion of the request from client to resource format. + """ + base = basic_batch_get_item_response(items) + additional_keys = { + "UnprocessedKeys": { + INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME: { + "Keys": [{"partition_key": item["partition_key"]} for item in items] + } + }, + } + return {**base, **additional_keys} + + +def basic_batch_write_item_put_response(items): + """Get a batch_write_item response in resource (ddb) format for any items.""" + return { + "UnprocessedItems": {INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME: [{"PutRequest": {"Item": item}} for item in items]} + } + + +def exhaustive_batch_write_item_put_response(items): + """ + Get a batch_write_item response in resource (ddb) format for any items. + This is not intended to be a real response that DynamoDB would return, + but the response should contain additional attributes that DynamoDB could return. + This is only intended to exhaustively test the conversion of the request from client to resource format. + """ + base = basic_batch_write_item_put_response(items) + additional_keys = { + "ItemCollectionMetrics": { + INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME: [ + {"ItemCollectionKey": {"partition_key": items[-1]["partition_key"]}} + ] + }, + } + return {**base, **additional_keys} + + +def basic_transact_write_items_response(items): + """Get a transact_write_items response in resource (ddb) format for any items.""" + return { + "ItemCollectionMetrics": { + INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME: [ + {"ItemCollectionKey": {"partition_key": items[-1]["partition_key"]}} + ] + }, + } + + +def basic_transact_get_items_response(items): + """Get a transact_get_items response in resource (ddb) format for any items.""" + return {"Responses": [{"Item": item} for item in items]} + + +def basic_update_item_response(item): + """Get an update_item response in resource (ddb) format for any item.""" + return {"Attributes": item} + + +def basic_delete_item_response(item): + """Get a delete_item response in resource (ddb) format for any item.""" + return {"Attributes": item} diff --git a/DynamoDbEncryption/runtimes/python/test/unit/internal/__init__.py b/DynamoDbEncryption/runtimes/python/test/unit/internal/__init__.py new file mode 100644 index 000000000..f94fd12a2 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/test/unit/internal/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py new file mode 100644 index 000000000..d1187d03e --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py @@ -0,0 +1,724 @@ +import pytest + +from aws_dbesdk_dynamodb.internal.client_to_resource import ClientShapeToResourceShapeConverter +from aws_dbesdk_dynamodb.internal.condition_expression_builder import InternalDBESDKDynamoDBConditionExpressionBuilder + +from ...items import ( + complex_item_ddb, + complex_item_dict, + complex_key_ddb, + complex_key_dict, + simple_item_ddb, + simple_item_dict, + simple_key_ddb, + simple_key_dict, +) +from ...requests import ( + basic_batch_execute_statement_request_encrypted_table, + basic_batch_get_item_request_ddb, + basic_batch_get_item_request_dict, + basic_batch_write_item_delete_request_ddb, + basic_batch_write_item_delete_request_dict, + basic_batch_write_item_put_request_ddb, + basic_batch_write_item_put_request_dict, + basic_delete_item_request_ddb, + basic_delete_item_request_dict, + basic_execute_statement_request_encrypted_table, + basic_execute_transaction_request_encrypted_table, + basic_get_item_request_ddb, + basic_get_item_request_dict, + basic_put_item_request_ddb, + basic_put_item_request_dict, + basic_query_request_ddb, + basic_query_request_dict, + basic_scan_request_ddb, + basic_scan_request_dict, + basic_transact_get_item_request_ddb, + basic_transact_get_item_request_dict, + basic_transact_write_item_condition_check_request_ddb, + basic_transact_write_item_condition_check_request_dict, + basic_transact_write_item_delete_request_ddb, + basic_transact_write_item_delete_request_dict, + basic_transact_write_item_put_request_ddb, + basic_transact_write_item_put_request_dict, + basic_update_item_request_ddb_unsigned_attribute, + basic_update_item_request_dict_unsigned_attribute, + exhaustive_get_item_request_ddb, + exhaustive_get_item_request_dict, + exhaustive_put_item_request_ddb, + exhaustive_put_item_request_dict, + exhaustive_query_request_ddb, + exhaustive_query_request_dict, + exhaustive_scan_request_ddb, + exhaustive_scan_request_dict, +) +from ...responses import ( + basic_batch_get_item_response, + basic_batch_write_item_put_response, + basic_delete_item_response, + basic_get_item_response, + basic_put_item_response, + basic_query_response, + basic_scan_response, + basic_transact_get_items_response, + basic_transact_write_items_response, + basic_update_item_response, + exhaustive_batch_get_item_response, + exhaustive_batch_write_item_put_response, + exhaustive_get_item_response, + exhaustive_put_item_response, + exhaustive_query_response, + exhaustive_scan_response, +) + +client_to_resource_converter = ClientShapeToResourceShapeConverter() + + +@pytest.fixture(params=[True, False], ids=["complex_item", "simple_item"]) +def use_complex_item(request): + return request.param + + +@pytest.fixture +def test_ddb_item(use_complex_item): + """Get a single test item in the appropriate format for the client.""" + if use_complex_item: + return complex_item_ddb + return simple_item_ddb + + +@pytest.fixture +def test_dict_item(use_complex_item): + """Get a single test item in the appropriate format for the client.""" + if use_complex_item: + return complex_item_dict + return simple_item_dict + + +@pytest.fixture +def test_ddb_key(use_complex_item): + """Get a single test item in the appropriate format for the client.""" + if use_complex_item: + return complex_key_ddb + return simple_key_ddb + + +@pytest.fixture +def test_dict_key(use_complex_item): + """Get a single test item in the appropriate format for the client.""" + if use_complex_item: + return complex_key_dict + return simple_key_dict + + +@pytest.fixture(params=[True, False], ids=["exhaustive_request", "basic_request"]) +def use_exhaustive_request(request): + return request.param + + +@pytest.fixture +def test_put_item_request_ddb(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_put_item_request_ddb + return basic_put_item_request_ddb + + +@pytest.fixture +def test_put_item_request_dict(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_put_item_request_dict + return basic_put_item_request_dict + + +def test_GIVEN_test_put_item_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_put_item_request_ddb, test_put_item_request_dict, test_ddb_item, test_dict_item +): + # Given: Put item request + request = test_put_item_request_ddb(test_ddb_item) + # When: Converting to resource format + dict_item = client_to_resource_converter.put_item_request(request) + # Then: Returns dict value + # For exhaustive requests, we need to handle ConditionExpression separately + # since it keeps the original DDB-formatted string + expected_dict_request = test_put_item_request_dict(test_dict_item) + for key in dict_item.keys(): + if key != "ConditionExpression": + assert dict_item[key] == expected_dict_request[key] + + +@pytest.fixture +def test_put_item_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_put_item_response + return basic_put_item_response + + +def test_GIVEN_test_put_item_response_WHEN_client_to_resource_THEN_returns_dict_value( + test_put_item_response, test_ddb_key, test_dict_key +): + # Given: Put item response + response = test_put_item_response(test_ddb_key) + # When: Converting to resource format + dict_item = client_to_resource_converter.put_item_response(response) + # Then: Returns dict value + assert dict_item == test_put_item_response(test_dict_key) + + +@pytest.fixture +def test_get_item_request_ddb(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_get_item_request_ddb + return basic_get_item_request_ddb + + +@pytest.fixture +def test_get_item_request_dict(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_get_item_request_dict + return basic_get_item_request_dict + + +def test_GIVEN_test_get_item_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_get_item_request_ddb, test_get_item_request_dict, test_ddb_item, test_dict_item +): + # Given: Get item request + request = test_get_item_request_ddb(test_ddb_item) + # When: Converting to resource format + dict_item = client_to_resource_converter.get_item_request(request) + # Then: Returns dict value + assert dict_item == test_get_item_request_dict(test_dict_item) + + +@pytest.fixture +def test_get_item_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_get_item_response + return basic_get_item_response + + +def test_GIVEN_test_get_item_response_WHEN_client_to_resource_THEN_returns_dict_value( + test_get_item_response, test_ddb_item, test_dict_item +): + # Given: Get item response + response = test_get_item_response(test_ddb_item) + # When: Converting to resource format + dict_item = client_to_resource_converter.get_item_response(response) + # Then: Returns dict value + assert dict_item == test_get_item_response(test_dict_item) + + +@pytest.fixture +def test_query_request_ddb(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_query_request_ddb + return basic_query_request_ddb + + +@pytest.fixture +def test_query_request_dict(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_query_request_dict + return basic_query_request_dict + + +def test_GIVEN_test_query_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_query_request_ddb, test_query_request_dict, test_ddb_item, test_dict_item +): + # Given: Query request + request = test_query_request_ddb(test_ddb_item) + # When: Converting to resource format + dict_item = client_to_resource_converter.query_request(request) + # Then: Returns dict value + for key in dict_item.keys(): + if key == "KeyConditionExpression": + assert_condition_expressions_are_equal(test_query_request_dict(test_dict_item), dict_item, key) + else: + assert dict_item[key] == test_query_request_dict(test_dict_item)[key] + + +@pytest.fixture +def test_query_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_query_response + return basic_query_response + + +def test_GIVEN_test_query_response_WHEN_client_to_resource_THEN_returns_dict_value( + test_query_response, test_ddb_item, test_dict_item +): + # Given: Query response + response = test_query_response([test_ddb_item]) + # When: Converting to resource format + dict_item = client_to_resource_converter.query_response(response) + # Then: Returns dict value + assert dict_item == test_query_response([test_dict_item]) + + +def get_string_for_key_condition_expression( + key_condition_expression, expression_attribute_names, expression_attribute_values +): + """Get the string for the key condition expression.""" + if not isinstance(key_condition_expression, str): + built_expression = InternalDBESDKDynamoDBConditionExpressionBuilder().build_expression( + key_condition_expression, expression_attribute_names, expression_attribute_values + ) + key_condition_expression = built_expression.condition_expression + expression_attribute_names = built_expression.attribute_name_placeholders + expression_attribute_values = built_expression.attribute_value_placeholders + for expression_attribute_name, value in expression_attribute_names.items(): + key_condition_expression = key_condition_expression.replace(expression_attribute_name, str(value)) + for expression_attribute_value, value in expression_attribute_values.items(): + key_condition_expression = key_condition_expression.replace(expression_attribute_value, str(value)) + return key_condition_expression + + +def assert_condition_expressions_are_equal(expected_item, actual_item, key): + expected_key_condition_expression = get_string_for_key_condition_expression( + expected_item[key], + expected_item["ExpressionAttributeNames"] if "ExpressionAttributeNames" in expected_item else {}, + expected_item["ExpressionAttributeValues"] if "ExpressionAttributeValues" in expected_item else {}, + ) + actual_key_condition_expression = get_string_for_key_condition_expression( + actual_item[key], + actual_item["ExpressionAttributeNames"] if "ExpressionAttributeNames" in actual_item else {}, + actual_item["ExpressionAttributeValues"] if "ExpressionAttributeValues" in actual_item else {}, + ) + assert expected_key_condition_expression == actual_key_condition_expression + + +@pytest.fixture +def test_scan_request_ddb(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_scan_request_ddb + return basic_scan_request_ddb + + +@pytest.fixture +def test_scan_request_dict(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_scan_request_dict + return basic_scan_request_dict + + +def test_GIVEN_test_scan_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_scan_request_ddb, test_scan_request_dict, test_ddb_item, test_dict_item +): + # Given: Scan request + request = test_scan_request_ddb(test_ddb_item) + # When: Converting to resource format + dict_item = client_to_resource_converter.scan_request(request) + # Then: Returns dict value + assert dict_item == test_scan_request_dict(test_dict_item) + + +@pytest.fixture +def test_scan_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_scan_response + return basic_scan_response + + +def test_GIVEN_test_scan_response_WHEN_client_to_resource_THEN_returns_dict_value( + test_scan_response, test_ddb_item, test_dict_item, test_ddb_key, test_dict_key +): + # Given: Scan response + response = test_scan_response([test_ddb_item], [test_ddb_key]) + # When: Converting to resource format + dict_item = client_to_resource_converter.scan_response(response) + # Then: Returns dict value + assert dict_item == test_scan_response([test_dict_item], [test_dict_key]) + + +@pytest.fixture +def test_batch_get_item_request_ddb(): + return basic_batch_get_item_request_ddb + + +@pytest.fixture +def test_batch_get_item_request_dict(): + return basic_batch_get_item_request_dict + + +def test_GIVEN_test_batch_get_item_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_batch_get_item_request_ddb, test_batch_get_item_request_dict, test_ddb_item, test_dict_item +): + # Given: Batch get item request + request = test_batch_get_item_request_ddb([test_ddb_item]) + # When: Converting to resource format + dict_item = client_to_resource_converter.batch_get_item_request(request) + # Then: Returns dict value + assert dict_item == test_batch_get_item_request_dict([test_dict_item]) + + +@pytest.fixture +def test_batch_get_item_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_batch_get_item_response + return basic_batch_get_item_response + + +def test_GIVEN_test_batch_get_item_response_WHEN_client_to_resource_THEN_returns_dict_value( + test_batch_get_item_response, test_ddb_item, test_dict_item +): + # Given: Batch get item response + response = test_batch_get_item_response([test_ddb_item]) + # When: Converting to resource format + dict_item = client_to_resource_converter.batch_get_item_response(response) + # Then: Returns dict value + assert dict_item == test_batch_get_item_response([test_dict_item]) + + +@pytest.fixture +def test_batch_write_item_put_request_ddb(): + return basic_batch_write_item_put_request_ddb + + +@pytest.fixture +def test_batch_write_item_put_request_dict(): + return basic_batch_write_item_put_request_dict + + +def test_GIVEN_test_batch_write_item_put_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_batch_write_item_put_request_ddb, test_batch_write_item_put_request_dict, test_ddb_item, test_dict_item +): + # Given: Batch write item request + request = test_batch_write_item_put_request_ddb([test_ddb_item]) + # When: Converting to resource format + dict_item = client_to_resource_converter.batch_write_item_request(request) + # Then: Returns dict value + assert dict_item == test_batch_write_item_put_request_dict([test_dict_item]) + + +@pytest.fixture +def test_batch_write_item_delete_request_ddb(): + return basic_batch_write_item_delete_request_ddb + + +@pytest.fixture +def test_batch_write_item_delete_request_dict(): + return basic_batch_write_item_delete_request_dict + + +def test_GIVEN_test_batch_write_item_delete_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_batch_write_item_delete_request_ddb, test_batch_write_item_delete_request_dict, test_ddb_key, test_dict_key +): + # Given: Batch write item delete request + request = test_batch_write_item_delete_request_ddb([test_ddb_key]) + # When: Converting to resource format + dict_item = client_to_resource_converter.batch_write_item_request(request) + # Then: Returns dict value + assert dict_item == test_batch_write_item_delete_request_dict([test_dict_key]) + + +@pytest.fixture +def test_batch_write_item_put_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_batch_write_item_put_response + return basic_batch_write_item_put_response + + +def test_GIVEN_test_batch_write_item_put_response_WHEN_client_to_resource_THEN_returns_dict_value( + test_batch_write_item_put_response, test_ddb_item, test_dict_item +): + # Given: Batch write item put response + response = test_batch_write_item_put_response([test_ddb_item]) + # When: Converting to resource format + dict_item = client_to_resource_converter.batch_write_item_response(response) + # Then: Returns dict value + assert dict_item == test_batch_write_item_put_response([test_dict_item]) + + +@pytest.fixture +def test_transact_write_items_put_request_ddb(): + return basic_transact_write_item_put_request_ddb + + +@pytest.fixture +def test_transact_write_items_put_request_dict(): + return basic_transact_write_item_put_request_dict + + +def test_GIVEN_test_transact_write_items_put_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_transact_write_items_put_request_ddb, test_transact_write_items_put_request_dict, test_ddb_item, test_dict_item +): + # Given: Transact write item put request + request = test_transact_write_items_put_request_ddb([test_ddb_item]) + # When: Converting to resource format + dict_item = client_to_resource_converter.transact_write_items_request(request) + # Then: Returns dict value + assert dict_item == test_transact_write_items_put_request_dict([test_dict_item]) + + +@pytest.fixture +def test_transact_write_items_delete_request_ddb(): + return basic_transact_write_item_delete_request_ddb + + +@pytest.fixture +def test_transact_write_items_delete_request_dict(): + return basic_transact_write_item_delete_request_dict + + +def test_GIVEN_test_transact_write_items_delete_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_transact_write_items_delete_request_ddb, + test_transact_write_items_delete_request_dict, + test_ddb_key, + test_dict_key, +): + # Given: Transact write item delete request + request = test_transact_write_items_delete_request_ddb([test_ddb_key]) + # When: Converting to resource format + dict_item = client_to_resource_converter.transact_write_items_request(request) + # Then: Returns dict value + assert dict_item == test_transact_write_items_delete_request_dict([test_dict_key]) + + +@pytest.fixture +def test_transact_write_items_condition_check_request_ddb(): + return basic_transact_write_item_condition_check_request_ddb + + +@pytest.fixture +def test_transact_write_items_condition_check_request_dict(): + return basic_transact_write_item_condition_check_request_dict + + +def test_GIVEN_test_transact_write_items_condition_check_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_transact_write_items_condition_check_request_ddb, + test_transact_write_items_condition_check_request_dict, + test_ddb_key, + test_dict_key, +): + # Given: Transact write item condition check request + request = test_transact_write_items_condition_check_request_ddb([test_ddb_key]) + # When: Converting to resource format + dict_item = client_to_resource_converter.transact_write_items_request(request) + # Then: Returns dict value + assert dict_item == test_transact_write_items_condition_check_request_dict([test_dict_key]) + + +@pytest.fixture +def test_transact_write_items_response(): + return basic_transact_write_items_response + + +def test_GIVEN_test_transact_write_items_response_WHEN_client_to_resource_THEN_returns_dict_value( + test_transact_write_items_response, test_ddb_item, test_dict_item +): + # Given: Transact write items response + response = test_transact_write_items_response([test_ddb_item]) + # When: Converting to resource format + dict_item = client_to_resource_converter.transact_write_items_response(response) + # Then: Returns dict value + assert dict_item == test_transact_write_items_response([test_dict_item]) + + +@pytest.fixture +def test_transact_get_items_request_ddb(): + return basic_transact_get_item_request_ddb + + +@pytest.fixture +def test_transact_get_items_request_dict(): + return basic_transact_get_item_request_dict + + +def test_GIVEN_test_transact_get_items_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_transact_get_items_request_ddb, test_transact_get_items_request_dict, test_ddb_key, test_dict_key +): + # Given: Transact get items request + request = test_transact_get_items_request_ddb([test_ddb_key]) + # When: Converting to resource format + dict_item = client_to_resource_converter.transact_get_items_request(request) + # Then: Returns dict value + assert dict_item == test_transact_get_items_request_dict([test_dict_key]) + + +@pytest.fixture +def test_transact_get_items_response(): + return basic_transact_get_items_response + + +def test_GIVEN_test_transact_get_items_response_WHEN_client_to_resource_THEN_returns_dict_value( + test_transact_get_items_response, test_ddb_item, test_dict_item +): + # Given: Transact get items response + response = test_transact_get_items_response([test_ddb_item]) + # When: Converting to resource format + dict_item = client_to_resource_converter.transact_get_items_response(response) + # Then: Returns dict value + assert dict_item == test_transact_get_items_response([test_dict_item]) + + +@pytest.fixture +def test_update_item_request_ddb(): + # Select unsigned attribute without loss of generality; + # resource/client logic doesn't care about signed attributes + # TODO: Add exhaustive request + return basic_update_item_request_ddb_unsigned_attribute + + +@pytest.fixture +def test_update_item_request_dict(): + # Select unsigned attribute without loss of generality; + # resource/client logic doesn't care about signed attributes + # TODO: Add exhaustive request + return basic_update_item_request_dict_unsigned_attribute + + +def test_GIVEN_test_update_item_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_update_item_request_ddb, test_update_item_request_dict, test_ddb_item, test_dict_item +): + # Given: Update item request + request = test_update_item_request_ddb(test_ddb_item) + # When: Converting to resource format + dict_item = client_to_resource_converter.update_item_request(request) + # Then: Returns dict value + assert dict_item == test_update_item_request_dict(test_dict_item) + + +@pytest.fixture +def test_update_item_response(): + # TODO: Add exhaustive response + return basic_update_item_response + + +def test_GIVEN_test_update_item_response_WHEN_client_to_resource_THEN_returns_dict_value( + test_update_item_response, test_ddb_item, test_dict_item +): + # Given: Update item response + response = test_update_item_response(test_ddb_item) + # When: Converting to resource format + dict_item = client_to_resource_converter.update_item_response(response) + # Then: Returns dict value + assert dict_item == test_update_item_response(test_dict_item) + + +@pytest.fixture +def test_execute_statement_request(): + return basic_execute_statement_request_encrypted_table + + +def test_GIVEN_test_execute_statement_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_execute_statement_request, test_ddb_item, test_dict_item +): + # Given: Execute statement request + request = test_execute_statement_request(test_ddb_item) + # When: Converting to resource format + dict_item = client_to_resource_converter.execute_statement_request(request) + # Then: Returns dict value (here, request is not modified) + assert dict_item == test_execute_statement_request(test_dict_item) + + +def test_GIVEN_test_execute_statement_response_WHEN_client_to_resource_THEN_raises_NotImplementedError(): + # Given: Execute statement response + # TODO: this + ddb_response = {} + # When: Converting to resource format + resource_response = client_to_resource_converter.execute_statement_response(ddb_response) + # Then: Returns dict value + assert resource_response == {} + + +@pytest.fixture +def test_execute_transaction_request(): + return basic_execute_transaction_request_encrypted_table + + +def test_GIVEN_test_execute_transaction_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_execute_transaction_request, test_ddb_item, test_dict_item +): + # Given: Execute transaction request + request = test_execute_transaction_request(test_ddb_item) + # When: Converting to resource format + dict_item = client_to_resource_converter.execute_transaction_request(request) + # Then: Returns dict value (here, request is not modified) + assert dict_item == test_execute_transaction_request(test_dict_item) + + +def test_GIVEN_test_execute_transaction_response_WHEN_client_to_resource_THEN_returns_dict_value(): + # Given: Execute transaction response + # TODO: this + ddb_response = {} + # When: Converting to resource format + resource_response = client_to_resource_converter.execute_transaction_response(ddb_response) + # Then: Returns dict value + assert resource_response == {} + + +@pytest.fixture +def test_batch_execute_statement_request(): + return basic_batch_execute_statement_request_encrypted_table + + +def test_GIVEN_test_batch_execute_statement_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_batch_execute_statement_request, test_ddb_item, test_dict_item +): + # Given: Batch execute statement request + request = test_batch_execute_statement_request() + # When: Converting to resource format + dict_item = client_to_resource_converter.batch_execute_statement_request(request) + # Then: Returns dict value (here, request is not modified) + assert dict_item == test_batch_execute_statement_request() + + +def test_GIVEN_test_batch_execute_statement_response_WHEN_client_to_resource_THEN_raises_NotImplementedError(): + # Given: Batch execute statement response + # TODO: this + ddb_response = {} + # When: Converting to resource format + resource_response = client_to_resource_converter.batch_execute_statement_response(ddb_response) + # Then: Returns dict value + assert resource_response == {} + + +@pytest.fixture +def test_delete_item_request_ddb(): + return basic_delete_item_request_ddb + + +@pytest.fixture +def test_delete_item_request_dict(): + return basic_delete_item_request_dict + + +def test_GIVEN_test_delete_item_request_WHEN_client_to_resource_THEN_returns_dict_value( + test_delete_item_request_ddb, test_delete_item_request_dict, test_ddb_key, test_dict_key +): + # Given: Delete item request + request = test_delete_item_request_ddb(test_ddb_key) + # When: Converting to resource format + dict_item = client_to_resource_converter.delete_item_request(request) + # Then: Returns dict value + assert dict_item == test_delete_item_request_dict(test_dict_key) + + +@pytest.fixture +def test_delete_item_response(): + return basic_delete_item_response + + +def test_GIVEN_test_delete_item_response_WHEN_client_to_resource_THEN_returns_dict_value( + test_delete_item_response, test_ddb_item, test_dict_item +): + # Given: Delete item response + response = test_delete_item_response(test_ddb_item) + # When: Converting to resource format + dict_item = client_to_resource_converter.delete_item_response(response) + # Then: Returns dict value + assert dict_item == test_delete_item_response(test_dict_item) + + +# ruff: noqa: E501 +def test_GIVEN_request_with_neither_ExpressionAttributeValues_nor_ExpressionAttributeNames_WHEN_condition_handler_THEN_returns_identity_output(): + # Given: Request with neither ExpressionAttributeValues nor ExpressionAttributeNames + request = exhaustive_put_item_request_ddb(simple_item_ddb) + if "ExpressionAttributeValues" in request: + del request["ExpressionAttributeValues"] + if "ExpressionAttributeNames" in request: + del request["ExpressionAttributeNames"] + # When: Call condition_handler method + actual = client_to_resource_converter.condition_handler("ConditionExpression", request) + # Then: Returns "identity" output (input condition expression and no attribute names or values) + expected = request["ConditionExpression"], {}, {} + assert actual == expected diff --git a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py new file mode 100644 index 000000000..26c142b89 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py @@ -0,0 +1,1024 @@ +import pytest + +from aws_dbesdk_dynamodb.internal.condition_expression_builder import InternalDBESDKDynamoDBConditionExpressionBuilder +from aws_dbesdk_dynamodb.internal.resource_to_client import ResourceShapeToClientShapeConverter + +from ...constants import INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME +from ...items import ( + complex_item_ddb, + complex_item_dict, + complex_key_ddb, + complex_key_dict, + simple_item_ddb, + simple_item_dict, + simple_key_ddb, + simple_key_dict, +) +from ...requests import ( + basic_batch_execute_statement_request_encrypted_table, + basic_batch_get_item_request_ddb, + basic_batch_get_item_request_dict, + basic_batch_write_item_delete_request_ddb, + basic_batch_write_item_delete_request_dict, + basic_batch_write_item_put_request_ddb, + basic_batch_write_item_put_request_dict, + basic_delete_item_request_ddb, + basic_delete_item_request_dict, + basic_execute_statement_request_encrypted_table, + basic_execute_transaction_request_encrypted_table, + basic_get_item_request_ddb, + basic_get_item_request_dict, + basic_put_item_request_ddb, + basic_put_item_request_dict, + basic_query_request_ddb, + basic_query_request_dict, + basic_scan_request_ddb, + basic_scan_request_dict, + basic_transact_get_item_request_ddb, + basic_transact_get_item_request_dict, + basic_transact_write_item_condition_check_request_ddb, + basic_transact_write_item_condition_check_request_dict, + basic_transact_write_item_delete_request_ddb, + basic_transact_write_item_delete_request_dict, + basic_transact_write_item_put_request_ddb, + basic_transact_write_item_put_request_dict, + basic_update_item_request_ddb_unsigned_attribute, + basic_update_item_request_dict_unsigned_attribute, + exhaustive_get_item_request_ddb, + exhaustive_get_item_request_dict, + exhaustive_put_item_request_ddb, + exhaustive_put_item_request_dict, + exhaustive_query_request_ddb, + exhaustive_query_request_dict, + exhaustive_scan_request_ddb, + exhaustive_scan_request_dict, +) +from ...responses import ( + basic_batch_get_item_response, + basic_batch_write_item_put_response, + basic_delete_item_response, + basic_get_item_response, + basic_put_item_response, + basic_query_response, + basic_scan_response, + basic_transact_get_items_response, + basic_transact_write_items_response, + basic_update_item_response, + exhaustive_batch_get_item_response, + exhaustive_batch_write_item_put_response, + exhaustive_get_item_response, + exhaustive_put_item_response, + exhaustive_query_response, + exhaustive_scan_response, +) + +resource_to_client_converter = ResourceShapeToClientShapeConverter(table_name=INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME) + + +@pytest.fixture(params=[True, False], ids=["complex_item", "simple_item"]) +def use_complex_item(request): + return request.param + + +@pytest.fixture +def test_ddb_item(use_complex_item): + """Get a single test item in the appropriate format for the client.""" + if use_complex_item: + return complex_item_ddb + return simple_item_ddb + + +@pytest.fixture +def test_dict_item(use_complex_item): + """Get a single test item in the appropriate format for the client.""" + if use_complex_item: + return complex_item_dict + return simple_item_dict + + +@pytest.fixture +def test_ddb_key(use_complex_item): + """Get a single test item in the appropriate format for the client.""" + if use_complex_item: + return complex_key_ddb + return simple_key_ddb + + +@pytest.fixture +def test_dict_key(use_complex_item): + """Get a single test item in the appropriate format for the client.""" + if use_complex_item: + return complex_key_dict + return simple_key_dict + + +@pytest.fixture(params=[True, False], ids=["exhaustive_request", "basic_request"]) +def use_exhaustive_request(request): + return request.param + + +@pytest.fixture +def test_put_item_request_ddb(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_put_item_request_ddb + return basic_put_item_request_ddb + + +@pytest.fixture +def test_put_item_request_dict(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_put_item_request_dict + return basic_put_item_request_dict + + +def sort_dynamodb_json_lists(obj): + """ + Utility that recursively sorts all lists in a DynamoDB JSON-like structure. + DynamoDB JSON uses lists to represent sets, so strict equality can fail. + Sort lists to ensure consistent ordering when comparing expected and actual items. + """ + if isinstance(obj, dict): + return {k: sort_dynamodb_json_lists(v) for k, v in obj.items()} + elif isinstance(obj, list): + try: + a = sorted(obj) # Sort lists for consistent comparison + return a + except TypeError: + return obj # Not all lists are sortable; ex. complex_item_ddb's "list" attribute + return obj + + +def test_GIVEN_test_put_item_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_put_item_request_ddb, test_put_item_request_dict, test_ddb_item, test_dict_item +): + # Given: Put item request + request = test_put_item_request_dict(test_dict_item) + # When: Converting to resource format + ddb_item = resource_to_client_converter.put_item_request(request) + # Then: Returns dict value + # For exhaustive requests, we need to handle ConditionExpression separately + # since it keeps the original DDB-formatted string + expected_ddb_request = test_put_item_request_ddb(test_ddb_item) + + actual_ddb_request = sort_dynamodb_json_lists(ddb_item) + expected_ddb_request = sort_dynamodb_json_lists(expected_ddb_request) + + for key in actual_ddb_request.keys(): + if key != "ConditionExpression": + assert actual_ddb_request[key] == expected_ddb_request[key] + + +def test_GIVEN_put_item_request_without_table_name_WHEN_resource_to_client_THEN_raises_error( + test_put_item_request_dict, +): + # Given: ResourceShapeToClientShapeConverter without table name + resource_to_client_converter_without_table_name = ResourceShapeToClientShapeConverter(table_name=None) + # Given: Put item request without table name + # Then: Raises ValueError + with pytest.raises(ValueError): + # When: Converting to resource format + resource_to_client_converter_without_table_name.put_item_request(test_put_item_request_dict) + + +@pytest.fixture +def test_put_item_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_put_item_response + return basic_put_item_response + + +def test_GIVEN_test_put_item_response_WHEN_resource_to_client_THEN_returns_ddb_value( + test_put_item_response, test_ddb_key, test_dict_key +): + # Given: Put item response + response = test_put_item_response(test_dict_key) + # When: Converting to resource format + ddb_item = resource_to_client_converter.put_item_response(response) + # Then: Returns dict value + expected_ddb_response = test_put_item_response(test_ddb_key) + assert ddb_item == expected_ddb_response + + +@pytest.fixture +def test_get_item_request_ddb(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_get_item_request_ddb + return basic_get_item_request_ddb + + +@pytest.fixture +def test_get_item_request_dict(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_get_item_request_dict + return basic_get_item_request_dict + + +def test_GIVEN_test_get_item_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_get_item_request_ddb, test_get_item_request_dict, test_ddb_item, test_dict_item +): + # Given: Get item request + request = test_get_item_request_dict(test_dict_item) + # When: Converting to resource format + ddb_item = resource_to_client_converter.get_item_request(request) + # Then: Returns dict value + expected_ddb_request = test_get_item_request_ddb(test_ddb_item) + assert ddb_item == expected_ddb_request + + +def test_GIVEN_get_item_request_without_table_name_WHEN_resource_to_client_THEN_raises_error( + test_get_item_request_dict, +): + # Given: ResourceShapeToClientShapeConverter without table name + resource_to_client_converter_without_table_name = ResourceShapeToClientShapeConverter(table_name=None) + # Given: Get item request without table name + # Then: Raises ValueError + with pytest.raises(ValueError): + # When: Converting to resource format + resource_to_client_converter_without_table_name.get_item_request(test_get_item_request_dict) + + +@pytest.fixture +def test_get_item_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_get_item_response + return basic_get_item_response + + +def test_GIVEN_test_get_item_response_WHEN_resource_to_client_THEN_returns_ddb_value( + test_get_item_response, test_ddb_item, test_dict_item +): + # Given: Get item response + response = test_get_item_response(test_dict_item) + # When: Converting to resource format + ddb_item = resource_to_client_converter.get_item_response(response) + # Then: Returns dict value + expected_ddb_response = test_get_item_response(test_ddb_item) + if "Item" in ddb_item: + ddb_item["Item"] = sort_dynamodb_json_lists(ddb_item["Item"]) + expected_ddb_response["Item"] = sort_dynamodb_json_lists(expected_ddb_response["Item"]) + assert ddb_item == expected_ddb_response + + +@pytest.fixture +def test_query_request_ddb(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_query_request_ddb + return basic_query_request_ddb + + +@pytest.fixture +def test_query_request_dict(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_query_request_dict + return basic_query_request_dict + + +def test_GIVEN_test_query_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_query_request_ddb, test_query_request_dict, test_ddb_item, test_dict_item +): + # Given: Query request + dict_request = test_query_request_dict(test_dict_item) + # When: Converting to resource format + ddb_request = resource_to_client_converter.query_request(dict_request) + # Then: Returns ddb value + actual_ddb_request = ddb_request + expected_ddb_request = test_query_request_ddb(test_ddb_item) + + try: + for key in actual_ddb_request["ExpressionAttributeValues"].keys(): + actual_ddb_request["ExpressionAttributeValues"][key] = sort_dynamodb_json_lists( + actual_ddb_request["ExpressionAttributeValues"][key] + ) + except KeyError: + pass + + try: + for key in expected_ddb_request["ExpressionAttributeValues"].keys(): + expected_ddb_request["ExpressionAttributeValues"][key] = sort_dynamodb_json_lists( + expected_ddb_request["ExpressionAttributeValues"][key] + ) + except KeyError: + pass + + try: + for key in actual_ddb_request["QueryFilter"].keys(): + actual_ddb_request["QueryFilter"][key]["AttributeValueList"] = [ + sort_dynamodb_json_lists(item) for item in actual_ddb_request["QueryFilter"][key]["AttributeValueList"] + ] + except KeyError: + pass + + try: + for key in expected_ddb_request["QueryFilter"].keys(): + expected_ddb_request["QueryFilter"][key]["AttributeValueList"] = [ + sort_dynamodb_json_lists(item) + for item in expected_ddb_request["QueryFilter"][key]["AttributeValueList"] + ] + except KeyError: + pass + + try: + for key in actual_ddb_request["ExclusiveStartKey"].keys(): + actual_ddb_request["ExclusiveStartKey"][key] = sort_dynamodb_json_lists( + actual_ddb_request["ExclusiveStartKey"][key] + ) + except KeyError: + pass + + try: + for key in expected_ddb_request["ExclusiveStartKey"].keys(): + expected_ddb_request["ExclusiveStartKey"][key] = sort_dynamodb_json_lists( + expected_ddb_request["ExclusiveStartKey"][key] + ) + except KeyError: + pass + + try: + for key in actual_ddb_request["KeyConditions"].keys(): + actual_ddb_request["KeyConditions"][key]["AttributeValueList"] = [ + sort_dynamodb_json_lists(item) + for item in actual_ddb_request["KeyConditions"][key]["AttributeValueList"] + ] + except KeyError: + pass + + try: + for key in expected_ddb_request["KeyConditions"].keys(): + expected_ddb_request["KeyConditions"][key]["AttributeValueList"] = [ + sort_dynamodb_json_lists(item) + for item in expected_ddb_request["KeyConditions"][key]["AttributeValueList"] + ] + except KeyError: + pass + + for key in actual_ddb_request.keys(): + if key == "KeyConditionExpression": + assert_condition_expressions_are_equal(expected_ddb_request, actual_ddb_request, key) + elif key == "ExpressionAttributeValues": + # Any values in expected_ddb_request MUST be in actual_ddb_request, + # but not the other way around. + # actual_ddb_request will generate attribute symbols as needed, + # but any values in expected_ddb_request MUST be present in actual_ddb_request. + if key in expected_ddb_request: + for name, value in expected_ddb_request[key].items(): + assert name in actual_ddb_request[key] + assert actual_ddb_request[key][name] == value + else: + # Keys in actual_ddb_request don't need to be in expected_ddb_request. + pass + elif key == "ExpressionAttributeNames": + # Any keys in expected_ddb_request MUST be in actual_ddb_request, + # but not the other way around. + # actual_ddb_request will generate attribute symbols as needed, + # but any keys in expected_ddb_request MUST be present in actual_ddb_request. + if key in expected_ddb_request: + for name, value in expected_ddb_request[key].items(): + assert name in actual_ddb_request[key] + assert actual_ddb_request[key][name] == value + else: + # Keys in actual_ddb_request don't need to be in expected_ddb_request. + pass + else: + assert actual_ddb_request[key] == expected_ddb_request[key] + + +def test_GIVEN_query_request_without_table_name_WHEN_resource_to_client_THEN_raises_error(test_query_request_dict): + # Given: ResourceShapeToClientShapeConverter without table name + resource_to_client_converter_without_table_name = ResourceShapeToClientShapeConverter(table_name=None) + # Given: Query request without table name + # Then: Raises ValueError + with pytest.raises(ValueError): + # When: Converting to resource format + resource_to_client_converter_without_table_name.query_request(test_query_request_dict) + + +@pytest.fixture +def test_query_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_query_response + return basic_query_response + + +def test_GIVEN_test_query_response_WHEN_resource_to_client_THEN_returns_ddb_value( + test_query_response, test_ddb_item, test_dict_item +): + # Given: Query response + response = test_query_response([test_dict_item]) + # When: Converting to resource format + ddb_item = resource_to_client_converter.query_response(response) + # Then: Returns dict value + actual_ddb_response = ddb_item + actual_ddb_response["Items"] = [sort_dynamodb_json_lists(item) for item in actual_ddb_response["Items"]] + expected_ddb_response = test_query_response([test_ddb_item]) + expected_ddb_response["Items"] = [sort_dynamodb_json_lists(item) for item in expected_ddb_response["Items"]] + + assert actual_ddb_response == expected_ddb_response + + +def get_string_for_key_condition_expression( + key_condition_expression, expression_attribute_names, expression_attribute_values +): + """Get the string for the key condition expression.""" + if not isinstance(key_condition_expression, str): + built_expression = InternalDBESDKDynamoDBConditionExpressionBuilder().build_expression( + key_condition_expression, expression_attribute_names, expression_attribute_values + ) + key_condition_expression = built_expression.condition_expression + expression_attribute_names = built_expression.attribute_name_placeholders + expression_attribute_values = built_expression.attribute_value_placeholders + for expression_attribute_name, value in expression_attribute_names.items(): + key_condition_expression = key_condition_expression.replace(expression_attribute_name, str(value)) + for expression_attribute_value, value in expression_attribute_values.items(): + key_condition_expression = key_condition_expression.replace(expression_attribute_value, str(value)) + return key_condition_expression + + +def assert_condition_expressions_are_equal(expected_item, actual_item, key): + expected_key_condition_expression = get_string_for_key_condition_expression( + expected_item[key], + expected_item["ExpressionAttributeNames"] if "ExpressionAttributeNames" in expected_item else {}, + expected_item["ExpressionAttributeValues"] if "ExpressionAttributeValues" in expected_item else {}, + ) + actual_key_condition_expression = get_string_for_key_condition_expression( + actual_item[key], + actual_item["ExpressionAttributeNames"] if "ExpressionAttributeNames" in actual_item else {}, + actual_item["ExpressionAttributeValues"] if "ExpressionAttributeValues" in actual_item else {}, + ) + assert expected_key_condition_expression == actual_key_condition_expression + + +@pytest.fixture +def test_scan_request_ddb(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_scan_request_ddb + return basic_scan_request_ddb + + +@pytest.fixture +def test_scan_request_dict(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_scan_request_dict + return basic_scan_request_dict + + +def sort_attribute_dynamodb_json_lists(item, attribute): + if attribute in item: + item[attribute] = sort_dynamodb_json_lists(item[attribute]) + return item + + +def sort_attribute_list_of_dynamodb_json_lists(item, attribute): + if attribute in item: + item[attribute] = [sort_dynamodb_json_lists(item) for item in item[attribute]] + return item + + +def test_GIVEN_test_scan_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_scan_request_ddb, test_scan_request_dict, test_ddb_item, test_dict_item +): + # Given: Scan request + request = test_scan_request_dict(test_dict_item) + # When: Converting to resource format + actual_ddb_request = resource_to_client_converter.scan_request(request) + # Then: Returns dict value + expected_ddb_request = test_scan_request_ddb(test_ddb_item) + + actual_ddb_request = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_request, "ScanFilter") + expected_ddb_request = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_request, "ScanFilter") + + actual_ddb_request = sort_attribute_dynamodb_json_lists(actual_ddb_request, "ExclusiveStartKey") + expected_ddb_request = sort_attribute_dynamodb_json_lists(expected_ddb_request, "ExclusiveStartKey") + + actual_ddb_request = sort_attribute_dynamodb_json_lists(actual_ddb_request, "ExpressionAttributeValues") + expected_ddb_request = sort_attribute_dynamodb_json_lists(expected_ddb_request, "ExpressionAttributeValues") + + assert actual_ddb_request == expected_ddb_request + + +def test_GIVEN_scan_request_without_table_name_WHEN_resource_to_client_THEN_raises_error(test_scan_request_dict): + # Given: ResourceShapeToClientShapeConverter without table name + resource_to_client_converter_without_table_name = ResourceShapeToClientShapeConverter(table_name=None) + # Given: Scan request without table name + # Then: Raises ValueError + with pytest.raises(ValueError): + # When: Converting to resource format + resource_to_client_converter_without_table_name.scan_request(test_scan_request_dict) + + +@pytest.fixture +def test_scan_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_scan_response + return basic_scan_response + + +def test_GIVEN_test_scan_response_WHEN_resource_to_client_THEN_returns_ddb_value( + test_scan_response, test_ddb_item, test_dict_item, test_ddb_key, test_dict_key +): + # Given: Scan response + response = test_scan_response([test_dict_item], [test_dict_key]) + # When: Converting to resource format + actual_ddb_response = resource_to_client_converter.scan_response(response) + # Then: Returns dict value + expected_ddb_response = test_scan_response([test_ddb_item], [test_ddb_key]) + + actual_ddb_response = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_response, "Items") + expected_ddb_response = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_response, "Items") + + assert actual_ddb_response == expected_ddb_response + + +@pytest.fixture +def test_batch_get_item_request_ddb(): + return basic_batch_get_item_request_ddb + + +@pytest.fixture +def test_batch_get_item_request_dict(): + return basic_batch_get_item_request_dict + + +def test_GIVEN_test_batch_get_item_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_batch_get_item_request_ddb, test_batch_get_item_request_dict, test_ddb_item, test_dict_item +): + # Given: Batch get item request + request = test_batch_get_item_request_dict([test_dict_item]) + # When: Converting to resource format + actual_ddb_request = resource_to_client_converter.batch_get_item_request(request) + # Then: Returns dict value + expected_ddb_request = test_batch_get_item_request_ddb([test_ddb_item]) + + actual_ddb_request = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_request, "RequestItems") + expected_ddb_request = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_request, "RequestItems") + + assert actual_ddb_request == expected_ddb_request + + +@pytest.fixture +def test_batch_get_item_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_batch_get_item_response + return basic_batch_get_item_response + + +def test_GIVEN_test_batch_get_item_response_WHEN_resource_to_client_THEN_returns_ddb_value( + test_batch_get_item_response, test_ddb_item, test_dict_item +): + # Given: Batch get item response + response = test_batch_get_item_response([test_dict_item]) + # When: Converting to resource format + actual_ddb_response = resource_to_client_converter.batch_get_item_response(response) + # Then: Returns dict value + expected_ddb_response = test_batch_get_item_response([test_ddb_item]) + + actual_ddb_response = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_response, "Responses") + expected_ddb_response = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_response, "Responses") + + assert actual_ddb_response == expected_ddb_response + + +@pytest.fixture +def test_batch_write_item_put_request_ddb(): + return basic_batch_write_item_put_request_ddb + + +@pytest.fixture +def test_batch_write_item_put_request_dict(): + return basic_batch_write_item_put_request_dict + + +def test_GIVEN_test_batch_write_item_put_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_batch_write_item_put_request_ddb, test_batch_write_item_put_request_dict, test_ddb_item, test_dict_item +): + # Given: Batch write item request + request = test_batch_write_item_put_request_dict([test_dict_item]) + # When: Converting to resource format + actual_ddb_request = resource_to_client_converter.batch_write_item_request(request) + # Then: Returns dict value + expected_ddb_request = test_batch_write_item_put_request_ddb([test_ddb_item]) + + actual_ddb_request = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_request, "RequestItems") + expected_ddb_request = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_request, "RequestItems") + + assert actual_ddb_request == expected_ddb_request + + +@pytest.fixture +def test_batch_write_item_delete_request_ddb(): + return basic_batch_write_item_delete_request_ddb + + +@pytest.fixture +def test_batch_write_item_delete_request_dict(): + return basic_batch_write_item_delete_request_dict + + +def test_GIVEN_test_batch_write_item_delete_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_batch_write_item_delete_request_ddb, test_batch_write_item_delete_request_dict, test_ddb_key, test_dict_key +): + # Given: Batch write item delete request + request = test_batch_write_item_delete_request_dict([test_dict_key]) + # When: Converting to resource format + actual_ddb_request = resource_to_client_converter.batch_write_item_request(request) + # Then: Returns dict value + expected_ddb_request = test_batch_write_item_delete_request_ddb([test_ddb_key]) + + actual_ddb_request = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_request, "RequestItems") + expected_ddb_request = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_request, "RequestItems") + + assert actual_ddb_request == expected_ddb_request + + +@pytest.fixture +def test_batch_write_item_put_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_batch_write_item_put_response + return basic_batch_write_item_put_response + + +def test_GIVEN_test_batch_write_item_put_response_WHEN_resource_to_client_THEN_returns_ddb_value( + test_batch_write_item_put_response, test_ddb_item, test_dict_item +): + # Given: Batch write item put response + response = test_batch_write_item_put_response([test_dict_item]) + # When: Converting to resource format + actual_ddb_response = resource_to_client_converter.batch_write_item_response(response) + # Then: Returns dict value + expected_ddb_response = test_batch_write_item_put_response([test_ddb_item]) + + actual_ddb_response = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_response, "UnprocessedItems") + expected_ddb_response = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_response, "UnprocessedItems") + + assert actual_ddb_response == expected_ddb_response + + +@pytest.fixture +def test_transact_write_items_put_request_ddb(): + return basic_transact_write_item_put_request_ddb + + +@pytest.fixture +def test_transact_write_items_put_request_dict(): + return basic_transact_write_item_put_request_dict + + +def test_GIVEN_test_transact_write_items_put_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_transact_write_items_put_request_ddb, test_transact_write_items_put_request_dict, test_ddb_item, test_dict_item +): + # Given: Transact write item put request + request = test_transact_write_items_put_request_dict([test_dict_item]) + # When: Converting to resource format + actual_ddb_request = resource_to_client_converter.transact_write_items_request(request) + # Then: Returns dict value + expected_ddb_request = test_transact_write_items_put_request_ddb([test_ddb_item]) + + actual_ddb_request = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_request, "TransactItems") + expected_ddb_request = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_request, "TransactItems") + + assert actual_ddb_request == expected_ddb_request + + +@pytest.fixture +def test_transact_write_items_delete_request_ddb(): + return basic_transact_write_item_delete_request_ddb + + +@pytest.fixture +def test_transact_write_items_delete_request_dict(): + return basic_transact_write_item_delete_request_dict + + +def test_GIVEN_test_transact_write_items_delete_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_transact_write_items_delete_request_ddb, + test_transact_write_items_delete_request_dict, + test_ddb_key, + test_dict_key, +): + # Given: Transact write item delete request + request = test_transact_write_items_delete_request_dict([test_dict_key]) + # When: Converting to resource format + actual_ddb_request = resource_to_client_converter.transact_write_items_request(request) + # Then: Returns dict value + expected_ddb_request = test_transact_write_items_delete_request_ddb([test_ddb_key]) + + actual_ddb_request = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_request, "TransactItems") + expected_ddb_request = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_request, "TransactItems") + + assert actual_ddb_request == expected_ddb_request + + +@pytest.fixture +def test_transact_write_items_condition_check_request_ddb(): + return basic_transact_write_item_condition_check_request_ddb + + +@pytest.fixture +def test_transact_write_items_condition_check_request_dict(): + return basic_transact_write_item_condition_check_request_dict + + +def test_GIVEN_test_transact_write_items_condition_check_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_transact_write_items_condition_check_request_ddb, + test_transact_write_items_condition_check_request_dict, + test_ddb_key, + test_dict_key, +): + # Given: Transact write item condition check request + request = test_transact_write_items_condition_check_request_dict([test_dict_key]) + # When: Converting to resource format + actual_ddb_request = resource_to_client_converter.transact_write_items_request(request) + # Then: Returns dict value + expected_ddb_request = test_transact_write_items_condition_check_request_ddb([test_ddb_key]) + + actual_ddb_request = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_request, "TransactItems") + expected_ddb_request = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_request, "TransactItems") + + assert actual_ddb_request == expected_ddb_request + + +@pytest.fixture +def test_transact_write_items_response(): + return basic_transact_write_items_response + + +def test_GIVEN_test_transact_write_items_response_WHEN_resource_to_client_THEN_returns_ddb_value( + test_transact_write_items_response, test_ddb_item, test_dict_item +): + # Given: Transact write items response + response = test_transact_write_items_response([test_dict_item]) + # When: Converting to resource format + actual_ddb_response = resource_to_client_converter.transact_write_items_response(response) + # Then: Returns dict value + expected_ddb_response = test_transact_write_items_response([test_ddb_item]) + + actual_ddb_response = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_response, "ConsumedCapacity") + expected_ddb_response = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_response, "ConsumedCapacity") + + assert actual_ddb_response == expected_ddb_response + + +@pytest.fixture +def test_transact_get_items_request_ddb(): + return basic_transact_get_item_request_ddb + + +@pytest.fixture +def test_transact_get_items_request_dict(): + return basic_transact_get_item_request_dict + + +def test_GIVEN_test_transact_get_items_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_transact_get_items_request_ddb, test_transact_get_items_request_dict, test_ddb_key, test_dict_key +): + # Given: Transact get items request + request = test_transact_get_items_request_dict([test_dict_key]) + # When: Converting to resource format + actual_ddb_request = resource_to_client_converter.transact_get_items_request(request) + # Then: Returns dict value + expected_ddb_request = test_transact_get_items_request_ddb([test_ddb_key]) + + actual_ddb_request = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_request, "TransactItems") + expected_ddb_request = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_request, "TransactItems") + + assert actual_ddb_request == expected_ddb_request + + +@pytest.fixture +def test_transact_get_items_response(): + return basic_transact_get_items_response + + +def test_GIVEN_test_transact_get_items_response_WHEN_resource_to_client_THEN_returns_ddb_value( + test_transact_get_items_response, test_ddb_item, test_dict_item +): + # Given: Transact get items response + response = test_transact_get_items_response([test_dict_item]) + # When: Converting to resource format + actual_ddb_response = resource_to_client_converter.transact_get_items_response(response) + # Then: Returns dict value + expected_ddb_response = test_transact_get_items_response([test_ddb_item]) + + actual_ddb_response = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_response, "Responses") + expected_ddb_response = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_response, "Responses") + + assert actual_ddb_response == expected_ddb_response + + +@pytest.fixture +def test_update_item_request_ddb(): + # Select unsigned attribute without loss of generality; + # resource/client logic doesn't care about signed attributes + # TODO: Add exhaustive request + return basic_update_item_request_ddb_unsigned_attribute + + +@pytest.fixture +def test_update_item_request_dict(): + # Select unsigned attribute without loss of generality; + # resource/client logic doesn't care about signed attributes + # TODO: Add exhaustive request + return basic_update_item_request_dict_unsigned_attribute + + +def test_GIVEN_test_update_item_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_update_item_request_ddb, test_update_item_request_dict, test_ddb_item, test_dict_item +): + # Given: Update item request + request = test_update_item_request_dict(test_dict_item) + # When: Converting to resource format + actual_ddb_request = resource_to_client_converter.update_item_request(request) + # Then: Returns dict value + expected_ddb_request = test_update_item_request_ddb(test_ddb_item) + + actual_ddb_request = sort_dynamodb_json_lists(actual_ddb_request) + expected_ddb_request = sort_dynamodb_json_lists(expected_ddb_request) + + assert actual_ddb_request == expected_ddb_request + + +def test_GIVEN_update_item_request_without_table_name_WHEN_resource_to_client_THEN_raises_error( + test_update_item_request_dict, +): + # Given: ResourceShapeToClientShapeConverter without table name + resource_to_client_converter_without_table_name = ResourceShapeToClientShapeConverter(table_name=None) + # Given: Put item request without table name + # Then: Raises ValueError + with pytest.raises(ValueError): + # When: Converting to resource format + resource_to_client_converter_without_table_name.update_item_request(test_update_item_request_dict) + + +@pytest.fixture +def test_update_item_response(): + # TODO: Add exhaustive response + return basic_update_item_response + + +def test_GIVEN_update_item_response_WHEN_resource_to_client_THEN_returns_dict_value( + test_update_item_response, test_ddb_item, test_dict_item +): + # Given: Update item response + response = test_update_item_response(test_dict_item) + # When: Converting to resource format + actual_ddb_response = resource_to_client_converter.update_item_response(response) + # Then: Returns dict value + expected_ddb_response = test_update_item_response(test_ddb_item) + + actual_ddb_response = sort_dynamodb_json_lists(actual_ddb_response["Attributes"]) + expected_ddb_response = sort_dynamodb_json_lists(expected_ddb_response["Attributes"]) + + assert actual_ddb_response == expected_ddb_response + + +@pytest.fixture +def test_execute_statement_request(): + return basic_execute_statement_request_encrypted_table + + +def test_GIVEN_test_execute_statement_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_execute_statement_request, test_ddb_item, test_dict_item +): + # Given: Execute statement request + request = test_execute_statement_request(test_dict_item) + # When: Converting to resource format + actual_ddb_request = resource_to_client_converter.execute_statement_request(request) + # Then: Returns dict value (here, request is not modified) + assert actual_ddb_request == test_execute_statement_request(test_ddb_item) + + +def test_GIVEN_test_execute_statement_response_WHEN_resource_to_client_THEN_returns_dict_value(): + # Given: Execute statement response + # TODO: this + dict_response = {} + # When: Converting to resource format + ddb_response = resource_to_client_converter.execute_statement_response(dict_response) + # Then: Returns dict value + assert ddb_response == {} + + +@pytest.fixture +def test_execute_transaction_request(): + return basic_execute_transaction_request_encrypted_table + + +def test_GIVEN_test_execute_transaction_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_execute_transaction_request, test_ddb_item, test_dict_item +): + # Given: Execute transaction request + request = test_execute_transaction_request(test_dict_item) + # When: Converting to resource format + actual_ddb_request = resource_to_client_converter.execute_transaction_request(request) + # Then: Returns dict value (here, request is not modified) + assert actual_ddb_request == test_execute_transaction_request(test_ddb_item) + + +def test_GIVEN_test_execute_transaction_response_WHEN_resource_to_client_THEN_returns_dict_value(): + # Given: Execute transaction response + # TODO: this + dict_response = {} + # When: Converting to resource format + ddb_response = resource_to_client_converter.execute_transaction_response(dict_response) + # Then: Returns dict value + assert ddb_response == {} + + +@pytest.fixture +def test_batch_execute_statement_request(): + return basic_batch_execute_statement_request_encrypted_table + + +def test_GIVEN_test_batch_execute_statement_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_batch_execute_statement_request, +): + # Given: Batch execute statement request + request = test_batch_execute_statement_request() + # When: Converting to resource format + actual_ddb_request = resource_to_client_converter.batch_execute_statement_request(request) + # Then: Returns dict value (here, request is not modified) + assert actual_ddb_request == test_batch_execute_statement_request() + + +def test_GIVEN_test_batch_execute_statement_response_WHEN_resource_to_client_THEN_returns_dict_value(): + # Given: Batch execute statement response + # TODO: this + dict_response = {} + # When: Converting to resource format + ddb_response = resource_to_client_converter.batch_execute_statement_response(dict_response) + # Then: Returns dict value + assert ddb_response == {} + + +@pytest.fixture +def test_delete_item_request_ddb(): + return basic_delete_item_request_ddb + + +@pytest.fixture +def test_delete_item_request_dict(): + return basic_delete_item_request_dict + + +def test_GIVEN_test_delete_item_request_WHEN_resource_to_client_THEN_returns_ddb_value( + test_delete_item_request_ddb, test_delete_item_request_dict, test_ddb_item, test_dict_item +): + # Given: Delete item request + request = test_delete_item_request_dict(test_dict_item) + # When: Converting to resource format + actual_ddb_request = resource_to_client_converter.delete_item_request(request) + # Then: Returns dict value + assert actual_ddb_request == test_delete_item_request_ddb(test_ddb_item) + + +def test_GIVEN_delete_item_request_without_table_name_WHEN_resource_to_client_THEN_raises_error( + test_delete_item_request_dict, +): + # Given: ResourceShapeToClientShapeConverter without table name + resource_to_client_converter_without_table_name = ResourceShapeToClientShapeConverter(table_name=None) + # Given: Put item request without table name + # Then: Raises ValueError + with pytest.raises(ValueError): + # When: Converting to resource format + resource_to_client_converter_without_table_name.delete_item_request(test_delete_item_request_dict) + + +@pytest.fixture +def test_delete_item_response(): + return basic_delete_item_response + + +def test_GIVEN_delete_item_response_WHEN_resource_to_client_THEN_returns_ddb_value( + test_delete_item_response, test_ddb_item, test_dict_item +): + # Given: Delete item response + response = test_delete_item_response(test_dict_item) + # When: Converting to resource format + actual_ddb_response = resource_to_client_converter.delete_item_response(response) + # Then: Returns dict value + expected_ddb_response = test_delete_item_response(test_ddb_item) + + actual_ddb_response = sort_dynamodb_json_lists(actual_ddb_response["Attributes"]) + expected_ddb_response = sort_dynamodb_json_lists(expected_ddb_response["Attributes"]) + + assert actual_ddb_response == expected_ddb_response + + +# ruff: noqa: E501 +def test_GIVEN_request_with_neither_ExpressionAttributeValues_nor_ExpressionAttributeNames_WHEN_condition_handler_THEN_returns_BuiltConditionExpression_output(): + # Given: Request with neither ExpressionAttributeValues nor ExpressionAttributeNames + request = exhaustive_put_item_request_dict(simple_item_dict) + if "ExpressionAttributeValues" in request: + del request["ExpressionAttributeValues"] + if "ExpressionAttributeNames" in request: + del request["ExpressionAttributeNames"] + actual = resource_to_client_converter.condition_handler("ConditionExpression", request) + # Reset expression_builder numbering to make test equality easier + # (ex. Instead of starting names at '#n2', it starts at '#n0' + # and can equal the `actual` expression string that starts at '#n0') + resource_to_client_converter.expression_builder.reset() + expected = resource_to_client_converter.expression_builder.build_expression(request["ConditionExpression"], {}, {}) + + assert actual == ( + expected.condition_expression, + expected.attribute_name_placeholders, + expected.attribute_value_placeholders, + ) From a9da8b255b8cb797467891eef988ea8726938793 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 27 May 2025 09:49:52 -0700 Subject: [PATCH 2/9] sync --- .../internal/client_to_resource.py | 49 ++-- .../internal/condition_expression_builder.py | 266 ++++++++++++++---- .../internal/resource_to_client.py | 32 +-- .../runtimes/python/test/requests.py | 4 +- .../unit/internal/test_client_to_resource.py | 21 +- .../unit/internal/test_resource_to_client.py | 68 +++-- 6 files changed, 294 insertions(+), 146 deletions(-) diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/client_to_resource.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/client_to_resource.py index 9665a55c2..ce56121ac 100644 --- a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/client_to_resource.py +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/client_to_resource.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 from aws_cryptography_internal_dynamodb.smithygenerated.com_amazonaws_dynamodb.boto3_conversions import ( InternalBoto3DynamoDBFormatConverter, ) @@ -7,6 +9,12 @@ class ClientShapeToResourceShapeConverter: def __init__(self, delete_table_name=True): + # Some callers expect the TableName kwarg to be removed from the outputs of this class. + # (EncryptedResource, EncryptedTable.) + # These callers' boto3 shapes do not include TableName. + # Other callers expect the TableName kwarg to be included in the outputs of this class. + # (EncryptedClient, EncryptedPaginator.) + # These callers' boto3 shapes include TableName. self.delete_table_name = delete_table_name self.boto3_converter = InternalBoto3DynamoDBFormatConverter( item_handler=TypeDeserializer().deserialize, condition_handler=self.condition_handler @@ -16,14 +24,13 @@ def condition_handler(self, expression_key, request): """Returns the input condition/names/values as-is.""" # Conditions do not need to be converted from strings to boto3 Attrs. # Resources accept either strings or Attrs. + # Return the provided condition string. condition = request[expression_key] - # This conversion in client_to_resource does not update neither - # ExpressionAttributeNames nor ExpressionAttributeValues. - # However, resource_to_client condition_handler may add new - # ExpressionAttributeNames and ExpressionAttributeValues. - # Smithy-generated code expects condition_handlers to return - # ExpressionAttributeNames and ExpressionAttributeValues. + # This conversion in client_to_resource does not update ExpressionAttributeNames or ExpressionAttributeValues. + # However, resource_to_client condition_handler may add new ExpressionAttributeNames and ExpressionAttributeValues. + # Smithy-generated code expects condition_handlers to return ExpressionAttributeNames and ExpressionAttributeValues, + # expecting empty dicts if there are none. try: names = request["ExpressionAttributeNames"] except KeyError: @@ -37,7 +44,7 @@ def condition_handler(self, expression_key, request): def put_item_request(self, put_item_request): out = self.boto3_converter.PutItemInput(put_item_request) - # put_item requests on a boto3.resource.Table do not have a table name. + # put_item requests on resources do not have a table name. if self.delete_table_name: del out["TableName"] return out @@ -47,7 +54,7 @@ def put_item_response(self, put_item_response): def get_item_request(self, get_item_request): out = self.boto3_converter.GetItemInput(get_item_request) - # get_item requests on a boto3.resource.Table do not have a table name. + # get_item requests on resources do not have a table name. if self.delete_table_name: del out["TableName"] return out @@ -57,7 +64,7 @@ def get_item_response(self, get_item_response): def query_request(self, query_request): out = self.boto3_converter.QueryInput(query_request) - # query requests on a boto3.resource.Table do not have a table name. + # query requests on resources do not have a table name. if self.delete_table_name: del out["TableName"] return out @@ -67,27 +74,33 @@ def query_response(self, query_response): def scan_request(self, scan_request): out = self.boto3_converter.ScanInput(scan_request) - # scan requests on a boto3.resource.Table do not have a table name. + # scan requests on resources do not have a table name. if self.delete_table_name: del out["TableName"] return out + def scan_response(self, scan_response): + return self.boto3_converter.ScanOutput(scan_response) + def delete_item_request(self, delete_item_request): out = self.boto3_converter.DeleteItemInput(delete_item_request) - # delete_item requests on a boto3.resource.Table do not have a table name. + # delete_item requests on resources do not have a table name. if self.delete_table_name: del out["TableName"] return out + + def delete_item_response(self, delete_item_response): + return self.boto3_converter.DeleteItemOutput(delete_item_response) def update_item_request(self, update_item_request): out = self.boto3_converter.UpdateItemInput(update_item_request) - # update_item requests on a boto3.resource.Table do not have a table name. + # update_item requests on resources do not have a table name. if self.delete_table_name: del out["TableName"] return out - - def scan_response(self, scan_response): - return self.boto3_converter.ScanOutput(scan_response) + + def update_item_response(self, update_item_response): + return self.boto3_converter.UpdateItemOutput(update_item_response) def transact_get_items_request(self, transact_get_items_request): return self.boto3_converter.TransactGetItemsInput(transact_get_items_request) @@ -113,18 +126,12 @@ def batch_write_item_request(self, batch_write_item_request): def batch_write_item_response(self, batch_write_item_response): return self.boto3_converter.BatchWriteItemOutput(batch_write_item_response) - def update_item_response(self, update_item_response): - return self.boto3_converter.UpdateItemOutput(update_item_response) - def batch_execute_statement_request(self, batch_execute_statement_request): return self.boto3_converter.BatchExecuteStatementInput(batch_execute_statement_request) def batch_execute_statement_response(self, batch_execute_statement_response): return self.boto3_converter.BatchExecuteStatementOutput(batch_execute_statement_response) - def delete_item_response(self, delete_item_response): - return self.boto3_converter.DeleteItemOutput(delete_item_response) - def execute_statement_request(self, execute_statement_request): return self.boto3_converter.ExecuteStatementInput(execute_statement_request) diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/condition_expression_builder.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/condition_expression_builder.py index 913d59f78..5f5b00e80 100644 --- a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/condition_expression_builder.py +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/condition_expression_builder.py @@ -17,25 +17,22 @@ class InternalDBESDKDynamoDBConditionExpressionBuilder: def __init__(self): self._name_count = 0 self._value_count = 0 - self._name_placeholder = "n" - self._value_placeholder = "v" + self._name_placeholder = 'n' + self._value_placeholder = 'v' def _get_name_placeholder(self): - return "#" + self._name_placeholder + str(self._name_count) + return '#' + self._name_placeholder + str(self._name_count) def _get_value_placeholder(self): - return ":" + self._value_placeholder + str(self._value_count) + return ':' + self._value_placeholder + str(self._value_count) def reset(self): """Resets the placeholder name and values""" self._name_count = 0 self._value_count = 0 - def build_expression( - self, condition, attribute_name_placeholders, attribute_value_placeholders, is_key_condition=False - ): - """ - Builds the condition expression and the dictionary of placeholders. + def build_expression(self, condition, is_key_condition=False): + """Builds the condition expression and the dictionary of placeholders. :type condition: ConditionBase :param condition: A condition to be built into a condition expression @@ -55,13 +52,14 @@ def build_expression( """ if not isinstance(condition, ConditionBase): raise DynamoDBNeedsConditionError(condition) + attribute_name_placeholders = {} + attribute_value_placeholders = {} condition_expression = self._build_expression( condition, attribute_name_placeholders, attribute_value_placeholders, is_key_condition=is_key_condition, ) - print(f"BuiltConditionExpression {condition_expression=}") return BuiltConditionExpression( condition_expression=condition_expression, attribute_name_placeholders=attribute_name_placeholders, @@ -77,7 +75,7 @@ def _build_expression( ): expression_dict = condition.get_expression() replaced_values = [] - for value in expression_dict["values"]: + for value in expression_dict['values']: # Build the necessary placeholders for that value. # Placeholders are built for both attribute names and values. replaced_value = self._build_expression_component( @@ -90,7 +88,9 @@ def _build_expression( replaced_values.append(replaced_value) # Fill out the expression using the operator and the # values that have been replaced with placeholders. - return expression_dict["format"].format(*replaced_values, operator=expression_dict["operator"]) + return expression_dict['format'].format( + *replaced_values, operator=expression_dict['operator'] + ) def _build_expression_component( self, @@ -115,15 +115,19 @@ def _build_expression_component( elif isinstance(value, AttributeBase): if is_key_condition and not isinstance(value, Key): raise DynamoDBNeedsKeyConditionError( - f"Attribute object {value.name} is of type {type(value)}. " - f"KeyConditionExpression only supports Attribute objects " - f"of type Key" + f'Attribute object {value.name} is of type {type(value)}. ' + f'KeyConditionExpression only supports Attribute objects ' + f'of type Key' ) - return self._build_name_placeholder(value, attribute_name_placeholders) + return self._build_name_placeholder( + value, attribute_name_placeholders + ) # If it is anything else, we treat it as a value and thus placeholders # are needed for the value. else: - return self._build_value_placeholder(value, attribute_value_placeholders, has_grouped_values) + return self._build_value_placeholder( + value, attribute_value_placeholders, has_grouped_values + ) def _build_name_placeholder(self, value, attribute_name_placeholders): attribute_name = value.name @@ -131,59 +135,207 @@ def _build_name_placeholder(self, value, attribute_name_placeholders): attribute_name_parts = ATTR_NAME_REGEX.findall(attribute_name) # Add a temporary placeholder for each of these parts. - placeholder_format = ATTR_NAME_REGEX.sub("%s", attribute_name) + placeholder_format = ATTR_NAME_REGEX.sub('%s', attribute_name) str_format_args = [] for part in attribute_name_parts: - # If the the name is already an AttributeName, use it. Don't make a new placeholder. - if part in attribute_name_placeholders: - str_format_args.append(part) - else: - name_placeholder = self._get_name_placeholder() - self._name_count += 1 - str_format_args.append(name_placeholder) - # Add the placeholder and value to dictionary of name placeholders. - attribute_name_placeholders[name_placeholder] = part + name_placeholder = self._get_name_placeholder() + self._name_count += 1 + str_format_args.append(name_placeholder) + # Add the placeholder and value to dictionary of name placeholders. + attribute_name_placeholders[name_placeholder] = part # Replace the temporary placeholders with the designated placeholders. return placeholder_format % tuple(str_format_args) - def _build_value_placeholder(self, value, attribute_value_placeholders, has_grouped_values=False): - print(f"{attribute_value_placeholders=}") + def _build_value_placeholder( + self, value, attribute_value_placeholders, has_grouped_values=False + ): # If the values are grouped, we need to add a placeholder for # each element inside of the actual value. - - # Also, you can define a grouped value with a colon here. - # If it's a colon, it's not a grouped value for the sake of this logic. - # Treat it as an "else" case. if has_grouped_values: placeholder_list = [] - # If it's a pre-defined grouped attribute, don't attempt to unpack it as if it were for v in value: - print(f"v1 {v=}") - # If the value is already an AttributeValue, reuse it. Don't make a new placeholder. - if v in attribute_value_placeholders: - print("in") - placeholder_list.append(v) - else: - print("not in") - value_placeholder = self._get_value_placeholder() - self._value_count += 1 - placeholder_list.append(value_placeholder) - attribute_value_placeholders[value_placeholder] = v + value_placeholder = self._get_value_placeholder() + self._value_count += 1 + placeholder_list.append(value_placeholder) + attribute_value_placeholders[value_placeholder] = v # Assuming the values are grouped by parenthesis. # IN is the currently the only one that uses this so it maybe # needed to be changed in future. - return "(" + ", ".join(placeholder_list) + ")" + return '(' + ', '.join(placeholder_list) + ')' # Otherwise, treat the value as a single value that needs only # one placeholder. else: - print(f"v2 {value=}") - # If the value is already an AttributeValue, reuse it. Don't make a new placeholder. - if value in attribute_value_placeholders: - print("in") - return value - else: - print("not in") - value_placeholder = self._get_value_placeholder() - self._value_count += 1 - attribute_value_placeholders[value_placeholder] = value - return value_placeholder + value_placeholder = self._get_value_placeholder() + self._value_count += 1 + attribute_value_placeholders[value_placeholder] = value + return value_placeholder + + +# class InternalDBESDKDynamoDBConditionExpressionBuilder: +# """This class is used to build condition expressions with placeholders""" + +# def __init__(self): +# self._name_count = 0 +# self._value_count = 0 +# self._name_placeholder = "n" +# self._value_placeholder = "v" + +# def _get_name_placeholder(self): +# return "#" + self._name_placeholder + str(self._name_count) + +# def _get_value_placeholder(self): +# return ":" + self._value_placeholder + str(self._value_count) + +# def reset(self): +# """Resets the placeholder name and values""" +# self._name_count = 0 +# self._value_count = 0 + +# def build_expression( +# self, condition, attribute_name_placeholders, attribute_value_placeholders, is_key_condition=False +# ): +# """ +# Builds the condition expression and the dictionary of placeholders. + +# :type condition: ConditionBase +# :param condition: A condition to be built into a condition expression +# string with any necessary placeholders. + +# :type is_key_condition: Boolean +# :param is_key_condition: True if the expression is for a +# KeyConditionExpression. False otherwise. + +# :rtype: (string, dict, dict) +# :returns: Will return a string representing the condition with +# placeholders inserted where necessary, a dictionary of +# placeholders for attribute names, and a dictionary of +# placeholders for attribute values. Here is a sample return value: + +# ('#n0 = :v0', {'#n0': 'myattribute'}, {':v1': 'myvalue'}) +# """ +# if not isinstance(condition, ConditionBase): +# raise DynamoDBNeedsConditionError(condition) +# condition_expression = self._build_expression( +# condition, +# attribute_name_placeholders, +# attribute_value_placeholders, +# is_key_condition=is_key_condition, +# ) +# return BuiltConditionExpression( +# condition_expression=condition_expression, +# attribute_name_placeholders=attribute_name_placeholders, +# attribute_value_placeholders=attribute_value_placeholders, +# ) + +# def _build_expression( +# self, +# condition, +# attribute_name_placeholders, +# attribute_value_placeholders, +# is_key_condition, +# ): +# expression_dict = condition.get_expression() +# replaced_values = [] +# for value in expression_dict["values"]: +# # Build the necessary placeholders for that value. +# # Placeholders are built for both attribute names and values. +# replaced_value = self._build_expression_component( +# value, +# attribute_name_placeholders, +# attribute_value_placeholders, +# condition.has_grouped_values, +# is_key_condition, +# ) +# replaced_values.append(replaced_value) +# # Fill out the expression using the operator and the +# # values that have been replaced with placeholders. +# return expression_dict["format"].format(*replaced_values, operator=expression_dict["operator"]) + +# def _build_expression_component( +# self, +# value, +# attribute_name_placeholders, +# attribute_value_placeholders, +# has_grouped_values, +# is_key_condition, +# ): +# # Continue to recurse if the value is a ConditionBase in order +# # to extract out all parts of the expression. +# if isinstance(value, ConditionBase): +# return self._build_expression( +# value, +# attribute_name_placeholders, +# attribute_value_placeholders, +# is_key_condition, +# ) +# # If it is not a ConditionBase, we can recurse no further. +# # So we check if it is an attribute and add placeholders for +# # its name +# elif isinstance(value, AttributeBase): +# if is_key_condition and not isinstance(value, Key): +# raise DynamoDBNeedsKeyConditionError( +# f"Attribute object {value.name} is of type {type(value)}. " +# f"KeyConditionExpression only supports Attribute objects " +# f"of type Key" +# ) +# return self._build_name_placeholder(value, attribute_name_placeholders) +# # If it is anything else, we treat it as a value and thus placeholders +# # are needed for the value. +# else: +# return self._build_value_placeholder(value, attribute_value_placeholders, has_grouped_values) + +# def _build_name_placeholder(self, value, attribute_name_placeholders): +# attribute_name = value.name +# # Figure out which parts of the attribute name that needs replacement. +# attribute_name_parts = ATTR_NAME_REGEX.findall(attribute_name) + +# # Add a temporary placeholder for each of these parts. +# placeholder_format = ATTR_NAME_REGEX.sub("%s", attribute_name) +# str_format_args = [] +# for part in attribute_name_parts: +# # If the the name is already an AttributeName, use it. Don't make a new placeholder. +# if part in attribute_name_placeholders: +# str_format_args.append(part) +# else: +# name_placeholder = self._get_name_placeholder() +# self._name_count += 1 +# str_format_args.append(name_placeholder) +# # Add the placeholder and value to dictionary of name placeholders. +# attribute_name_placeholders[name_placeholder] = part +# # Replace the temporary placeholders with the designated placeholders. +# return placeholder_format % tuple(str_format_args) + +# def _build_value_placeholder(self, value, attribute_value_placeholders, has_grouped_values=False): +# # If the values are grouped, we need to add a placeholder for +# # each element inside of the actual value. + +# # Also, you can define a grouped value with a colon here. +# # If it's a colon, it's not a grouped value for the sake of this logic. +# # Treat it as an "else" case. +# if has_grouped_values: +# placeholder_list = [] +# # If it's a pre-defined grouped attribute, don't attempt to unpack it as if it were +# for v in value: +# # If the value is already an AttributeValue, reuse it. Don't make a new placeholder. +# if v in attribute_value_placeholders: +# placeholder_list.append(v) +# else: +# value_placeholder = self._get_value_placeholder() +# self._value_count += 1 +# placeholder_list.append(value_placeholder) +# attribute_value_placeholders[value_placeholder] = v +# # Assuming the values are grouped by parenthesis. +# # IN is the currently the only one that uses this so it maybe +# # needed to be changed in future. +# return "(" + ", ".join(placeholder_list) + ")" +# # Otherwise, treat the value as a single value that needs only +# # one placeholder. +# else: +# # If the value is already an AttributeValue, reuse it. Don't make a new placeholder. +# if value in attribute_value_placeholders: +# return value +# else: +# value_placeholder = self._get_value_placeholder() +# self._value_count += 1 +# attribute_value_placeholders[value_placeholder] = value +# return value_placeholder diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/resource_to_client.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/resource_to_client.py index 93afbd9a0..996d2fd4c 100644 --- a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/resource_to_client.py +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/resource_to_client.py @@ -1,9 +1,10 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 from aws_cryptography_internal_dynamodb.smithygenerated.com_amazonaws_dynamodb.boto3_conversions import ( InternalBoto3DynamoDBFormatConverter, ) from boto3.dynamodb.types import TypeSerializer - -from aws_dbesdk_dynamodb.internal.condition_expression_builder import InternalDBESDKDynamoDBConditionExpressionBuilder +from boto3.dynamodb.conditions import ConditionExpressionBuilder class ResourceShapeToClientShapeConverter: @@ -12,8 +13,11 @@ def __init__(self, table_name=None): self.boto3_converter = InternalBoto3DynamoDBFormatConverter( item_handler=TypeSerializer().serialize, condition_handler=self.condition_handler ) + # TableName is optional; + # Some requests require it (ex. put_item, update_item, delete_item), + # but others do not (ex. transact_get_items, batch_write_item). self.table_name = table_name - self.expression_builder = InternalDBESDKDynamoDBConditionExpressionBuilder() + self.expression_builder = ConditionExpressionBuilder() def condition_handler(self, expression_key, request): """ @@ -40,30 +44,12 @@ def condition_handler(self, expression_key, request): and condition_expression.__module__ == "boto3.dynamodb.conditions" ): built_condition_expression = self.expression_builder.build_expression( - condition_expression, existing_expression_attribute_names, existing_expression_attribute_values + condition_expression ) + return built_condition_expression.condition_expression, built_condition_expression.attribute_name_placeholders, built_condition_expression.attribute_value_placeholders else: return condition_expression, existing_expression_attribute_names, existing_expression_attribute_values - # Unpack returned BuiltConditionExpression. - expression_str = built_condition_expression.condition_expression - attribute_names_from_built_expression = built_condition_expression.attribute_name_placeholders - # Join any placeholder ExpressionAttributeNames with any other ExpressionAttributeNames. - # The BuiltConditionExpression will return new names, not any that already exist. - # The two sets of names must be joined to form the complete set of names for the condition expression. - try: - out_names = request["ExpressionAttributeNames"] | attribute_names_from_built_expression - except KeyError: - out_names = attribute_names_from_built_expression - # Join existing and new values. - attribute_values_from_built_expression = built_condition_expression.attribute_value_placeholders - try: - out_values = request["ExpressionAttributeValues"] | attribute_values_from_built_expression - except KeyError: - out_values = attribute_values_from_built_expression - - return expression_str, out_names, out_values - def put_item_request(self, put_item_request): # put_item requests on a boto3.resource.Table require a configured table name. if not self.table_name: diff --git a/DynamoDbEncryption/runtimes/python/test/requests.py b/DynamoDbEncryption/runtimes/python/test/requests.py index 544af3ce8..ee46ef84c 100644 --- a/DynamoDbEncryption/runtimes/python/test/requests.py +++ b/DynamoDbEncryption/runtimes/python/test/requests.py @@ -158,8 +158,8 @@ def base_exhaustive_put_item_request(item): }, "sort_key": {"AttributeValueList": [item["sort_key"]], "ComparisonOperator": "EQ"}, }, - "ExpressionAttributeNames": {"#pk": "partition_key", "#sk": "sort_key"}, - "ExpressionAttributeValues": {":pk": item["partition_key"], ":sk": item["sort_key"]}, + # "ExpressionAttributeNames": {"#pk": "partition_key", "#sk": "sort_key"}, + # "ExpressionAttributeValues": {":pk": item["partition_key"], ":sk": item["sort_key"]}, "ReturnConsumedCapacity": "TOTAL", "ReturnItemCollectionMetrics": "SIZE", "ReturnValues": "ALL_OLD", diff --git a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py index d1187d03e..d6d521089 100644 --- a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py +++ b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 import pytest from aws_dbesdk_dynamodb.internal.client_to_resource import ClientShapeToResourceShapeConverter @@ -269,6 +271,10 @@ def get_string_for_key_condition_expression( key_condition_expression = key_condition_expression.replace(expression_attribute_name, str(value)) for expression_attribute_value, value in expression_attribute_values.items(): key_condition_expression = key_condition_expression.replace(expression_attribute_value, str(value)) + # Sometimes, the generated string has parentheses around the condition expression. + # It doesn't matter for the purposes of this test, so we remove them. + if key_condition_expression.startswith("(") and key_condition_expression.endswith(")"): + key_condition_expression = key_condition_expression[1:-1] return key_condition_expression @@ -707,18 +713,3 @@ def test_GIVEN_test_delete_item_response_WHEN_client_to_resource_THEN_returns_di dict_item = client_to_resource_converter.delete_item_response(response) # Then: Returns dict value assert dict_item == test_delete_item_response(test_dict_item) - - -# ruff: noqa: E501 -def test_GIVEN_request_with_neither_ExpressionAttributeValues_nor_ExpressionAttributeNames_WHEN_condition_handler_THEN_returns_identity_output(): - # Given: Request with neither ExpressionAttributeValues nor ExpressionAttributeNames - request = exhaustive_put_item_request_ddb(simple_item_ddb) - if "ExpressionAttributeValues" in request: - del request["ExpressionAttributeValues"] - if "ExpressionAttributeNames" in request: - del request["ExpressionAttributeNames"] - # When: Call condition_handler method - actual = client_to_resource_converter.condition_handler("ConditionExpression", request) - # Then: Returns "identity" output (input condition expression and no attribute names or values) - expected = request["ConditionExpression"], {}, {} - assert actual == expected diff --git a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py index 26c142b89..14db197dc 100644 --- a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py +++ b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py @@ -1,6 +1,8 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 import pytest -from aws_dbesdk_dynamodb.internal.condition_expression_builder import InternalDBESDKDynamoDBConditionExpressionBuilder +from boto3.dynamodb.conditions import ConditionExpressionBuilder from aws_dbesdk_dynamodb.internal.resource_to_client import ResourceShapeToClientShapeConverter from ...constants import INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME @@ -164,7 +166,33 @@ def test_GIVEN_test_put_item_request_WHEN_resource_to_client_THEN_returns_ddb_va expected_ddb_request = sort_dynamodb_json_lists(expected_ddb_request) for key in actual_ddb_request.keys(): - if key != "ConditionExpression": + if key == "ConditionExpression": + assert_condition_expressions_are_equal(expected_ddb_request, actual_ddb_request, key) + elif key == "ExpressionAttributeValues": + # Any values in expected_ddb_request MUST be in actual_ddb_request, + # but not the other way around. + # actual_ddb_request will generate attribute symbols as needed, + # but any values in expected_ddb_request MUST be present in actual_ddb_request. + if key in expected_ddb_request: + for name, value in expected_ddb_request[key].items(): + assert name in actual_ddb_request[key] + assert actual_ddb_request[key][name] == value + else: + # Keys in actual_ddb_request don't need to be in expected_ddb_request. + pass + elif key == "ExpressionAttributeNames": + # Any keys in expected_ddb_request MUST be in actual_ddb_request, + # but not the other way around. + # actual_ddb_request will generate attribute symbols as needed, + # but any keys in expected_ddb_request MUST be present in actual_ddb_request. + if key in expected_ddb_request: + for name, value in expected_ddb_request[key].items(): + assert name in actual_ddb_request[key] + assert actual_ddb_request[key][name] == value + else: + # Keys in actual_ddb_request don't need to be in expected_ddb_request. + pass + else: assert actual_ddb_request[key] == expected_ddb_request[key] @@ -420,7 +448,7 @@ def get_string_for_key_condition_expression( ): """Get the string for the key condition expression.""" if not isinstance(key_condition_expression, str): - built_expression = InternalDBESDKDynamoDBConditionExpressionBuilder().build_expression( + built_expression = ConditionExpressionBuilder().build_expression( key_condition_expression, expression_attribute_names, expression_attribute_values ) key_condition_expression = built_expression.condition_expression @@ -430,6 +458,10 @@ def get_string_for_key_condition_expression( key_condition_expression = key_condition_expression.replace(expression_attribute_name, str(value)) for expression_attribute_value, value in expression_attribute_values.items(): key_condition_expression = key_condition_expression.replace(expression_attribute_value, str(value)) + # Sometimes, the generated string has parentheses around the condition expression. + # It doesn't matter for the purposes of this test, so we remove them. + if key_condition_expression.startswith("(") and key_condition_expression.endswith(")"): + key_condition_expression = key_condition_expression[1:-1] return key_condition_expression @@ -444,6 +476,8 @@ def assert_condition_expressions_are_equal(expected_item, actual_item, key): actual_item["ExpressionAttributeNames"] if "ExpressionAttributeNames" in actual_item else {}, actual_item["ExpressionAttributeValues"] if "ExpressionAttributeValues" in actual_item else {}, ) + print(f"{expected_key_condition_expression=}") + print(f"{actual_key_condition_expression=}") assert expected_key_condition_expression == actual_key_condition_expression @@ -974,7 +1008,7 @@ def test_GIVEN_delete_item_request_without_table_name_WHEN_resource_to_client_TH ): # Given: ResourceShapeToClientShapeConverter without table name resource_to_client_converter_without_table_name = ResourceShapeToClientShapeConverter(table_name=None) - # Given: Put item request without table name + # Given: Delete item request without table name # Then: Raises ValueError with pytest.raises(ValueError): # When: Converting to resource format @@ -996,29 +1030,7 @@ def test_GIVEN_delete_item_response_WHEN_resource_to_client_THEN_returns_ddb_val # Then: Returns dict value expected_ddb_response = test_delete_item_response(test_ddb_item) - actual_ddb_response = sort_dynamodb_json_lists(actual_ddb_response["Attributes"]) - expected_ddb_response = sort_dynamodb_json_lists(expected_ddb_response["Attributes"]) + actual_ddb_response["Attributes"] = sort_dynamodb_json_lists(actual_ddb_response["Attributes"]) + expected_ddb_response["Attributes"] = sort_dynamodb_json_lists(expected_ddb_response["Attributes"]) assert actual_ddb_response == expected_ddb_response - - -# ruff: noqa: E501 -def test_GIVEN_request_with_neither_ExpressionAttributeValues_nor_ExpressionAttributeNames_WHEN_condition_handler_THEN_returns_BuiltConditionExpression_output(): - # Given: Request with neither ExpressionAttributeValues nor ExpressionAttributeNames - request = exhaustive_put_item_request_dict(simple_item_dict) - if "ExpressionAttributeValues" in request: - del request["ExpressionAttributeValues"] - if "ExpressionAttributeNames" in request: - del request["ExpressionAttributeNames"] - actual = resource_to_client_converter.condition_handler("ConditionExpression", request) - # Reset expression_builder numbering to make test equality easier - # (ex. Instead of starting names at '#n2', it starts at '#n0' - # and can equal the `actual` expression string that starts at '#n0') - resource_to_client_converter.expression_builder.reset() - expected = resource_to_client_converter.expression_builder.build_expression(request["ConditionExpression"], {}, {}) - - assert actual == ( - expected.condition_expression, - expected.attribute_name_placeholders, - expected.attribute_value_placeholders, - ) From 9270d73532b5bddf163d08a380d6b3505c1e08bd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 27 May 2025 10:13:30 -0700 Subject: [PATCH 3/9] sync --- .../aws_dbesdk_dynamodb/internal/client_to_resource.py | 10 ++++++---- .../aws_dbesdk_dynamodb/internal/resource_to_client.py | 10 ++++++---- DynamoDbEncryption/runtimes/python/test/items.py | 2 ++ DynamoDbEncryption/runtimes/python/test/requests.py | 4 +++- DynamoDbEncryption/runtimes/python/test/responses.py | 2 ++ .../test/unit/internal/test_client_to_resource.py | 4 ++-- .../test/unit/internal/test_resource_to_client.py | 2 +- 7 files changed, 22 insertions(+), 12 deletions(-) diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/client_to_resource.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/client_to_resource.py index ce56121ac..9c4244310 100644 --- a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/client_to_resource.py +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/client_to_resource.py @@ -28,8 +28,10 @@ def condition_handler(self, expression_key, request): condition = request[expression_key] # This conversion in client_to_resource does not update ExpressionAttributeNames or ExpressionAttributeValues. - # However, resource_to_client condition_handler may add new ExpressionAttributeNames and ExpressionAttributeValues. - # Smithy-generated code expects condition_handlers to return ExpressionAttributeNames and ExpressionAttributeValues, + # However, resource_to_client condition_handler may add new ExpressionAttributeNames and + # ExpressionAttributeValues. + # Smithy-generated code expects condition_handlers to return ExpressionAttributeNames and + # ExpressionAttributeValues, # expecting empty dicts if there are none. try: names = request["ExpressionAttributeNames"] @@ -88,7 +90,7 @@ def delete_item_request(self, delete_item_request): if self.delete_table_name: del out["TableName"] return out - + def delete_item_response(self, delete_item_response): return self.boto3_converter.DeleteItemOutput(delete_item_response) @@ -98,7 +100,7 @@ def update_item_request(self, update_item_request): if self.delete_table_name: del out["TableName"] return out - + def update_item_response(self, update_item_response): return self.boto3_converter.UpdateItemOutput(update_item_response) diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/resource_to_client.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/resource_to_client.py index 996d2fd4c..aa3f94ad7 100644 --- a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/resource_to_client.py +++ b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/resource_to_client.py @@ -3,8 +3,8 @@ from aws_cryptography_internal_dynamodb.smithygenerated.com_amazonaws_dynamodb.boto3_conversions import ( InternalBoto3DynamoDBFormatConverter, ) -from boto3.dynamodb.types import TypeSerializer from boto3.dynamodb.conditions import ConditionExpressionBuilder +from boto3.dynamodb.types import TypeSerializer class ResourceShapeToClientShapeConverter: @@ -43,10 +43,12 @@ def condition_handler(self, expression_key, request): hasattr(condition_expression, "__module__") and condition_expression.__module__ == "boto3.dynamodb.conditions" ): - built_condition_expression = self.expression_builder.build_expression( - condition_expression + built_condition_expression = self.expression_builder.build_expression(condition_expression) + return ( + built_condition_expression.condition_expression, + built_condition_expression.attribute_name_placeholders, + built_condition_expression.attribute_value_placeholders, ) - return built_condition_expression.condition_expression, built_condition_expression.attribute_name_placeholders, built_condition_expression.attribute_value_placeholders else: return condition_expression, existing_expression_attribute_names, existing_expression_attribute_values diff --git a/DynamoDbEncryption/runtimes/python/test/items.py b/DynamoDbEncryption/runtimes/python/test/items.py index fcdbf4278..2383585d6 100644 --- a/DynamoDbEncryption/runtimes/python/test/items.py +++ b/DynamoDbEncryption/runtimes/python/test/items.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 from decimal import Decimal simple_item_ddb = { diff --git a/DynamoDbEncryption/runtimes/python/test/requests.py b/DynamoDbEncryption/runtimes/python/test/requests.py index ee46ef84c..705b9ede1 100644 --- a/DynamoDbEncryption/runtimes/python/test/requests.py +++ b/DynamoDbEncryption/runtimes/python/test/requests.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Request constants for DynamoDB operations used for testing.""" from boto3.dynamodb.conditions import Attr, Key @@ -140,7 +142,7 @@ def basic_batch_execute_statement_request_plaintext_table(): # No exhaustive requests are intended to be able to be used as real requests. # Some parameters conflict with each other when sent to DynamoDB. -# These are only intended to test the conversion of the request from client to resource format. +# These are only intended to test the conversion of the structure from client to resource format. def base_exhaustive_put_item_request(item): diff --git a/DynamoDbEncryption/runtimes/python/test/responses.py b/DynamoDbEncryption/runtimes/python/test/responses.py index 6237385d2..9c4ba2a18 100644 --- a/DynamoDbEncryption/runtimes/python/test/responses.py +++ b/DynamoDbEncryption/runtimes/python/test/responses.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 from test.integ.encrypted.test_resource import INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME diff --git a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py index d6d521089..e3f7e2f02 100644 --- a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py +++ b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py @@ -3,7 +3,7 @@ import pytest from aws_dbesdk_dynamodb.internal.client_to_resource import ClientShapeToResourceShapeConverter -from aws_dbesdk_dynamodb.internal.condition_expression_builder import InternalDBESDKDynamoDBConditionExpressionBuilder +from boto3.dynamodb.conditions import ConditionExpressionBuilder from ...items import ( complex_item_ddb, @@ -261,7 +261,7 @@ def get_string_for_key_condition_expression( ): """Get the string for the key condition expression.""" if not isinstance(key_condition_expression, str): - built_expression = InternalDBESDKDynamoDBConditionExpressionBuilder().build_expression( + built_expression = ConditionExpressionBuilder().build_expression( key_condition_expression, expression_attribute_names, expression_attribute_values ) key_condition_expression = built_expression.condition_expression diff --git a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py index 14db197dc..4655ff914 100644 --- a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py +++ b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py @@ -1,8 +1,8 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 import pytest - from boto3.dynamodb.conditions import ConditionExpressionBuilder + from aws_dbesdk_dynamodb.internal.resource_to_client import ResourceShapeToClientShapeConverter from ...constants import INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME From b183d0b906cbc92dc45b6cd2a3a00cc78cba83d9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 27 May 2025 10:13:56 -0700 Subject: [PATCH 4/9] sync --- .../internal/condition_expression_builder.py | 341 ------------------ 1 file changed, 341 deletions(-) delete mode 100644 DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/condition_expression_builder.py diff --git a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/condition_expression_builder.py b/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/condition_expression_builder.py deleted file mode 100644 index 5f5b00e80..000000000 --- a/DynamoDbEncryption/runtimes/python/src/aws_dbesdk_dynamodb/internal/condition_expression_builder.py +++ /dev/null @@ -1,341 +0,0 @@ -# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -import re - -from boto3.dynamodb.conditions import AttributeBase, BuiltConditionExpression, ConditionBase, Key -from boto3.exceptions import ( - DynamoDBNeedsConditionError, - DynamoDBNeedsKeyConditionError, -) - -ATTR_NAME_REGEX = re.compile(r"[^.\[\]]+(?![^\[]*\])") - - -class InternalDBESDKDynamoDBConditionExpressionBuilder: - """This class is used to build condition expressions with placeholders""" - - def __init__(self): - self._name_count = 0 - self._value_count = 0 - self._name_placeholder = 'n' - self._value_placeholder = 'v' - - def _get_name_placeholder(self): - return '#' + self._name_placeholder + str(self._name_count) - - def _get_value_placeholder(self): - return ':' + self._value_placeholder + str(self._value_count) - - def reset(self): - """Resets the placeholder name and values""" - self._name_count = 0 - self._value_count = 0 - - def build_expression(self, condition, is_key_condition=False): - """Builds the condition expression and the dictionary of placeholders. - - :type condition: ConditionBase - :param condition: A condition to be built into a condition expression - string with any necessary placeholders. - - :type is_key_condition: Boolean - :param is_key_condition: True if the expression is for a - KeyConditionExpression. False otherwise. - - :rtype: (string, dict, dict) - :returns: Will return a string representing the condition with - placeholders inserted where necessary, a dictionary of - placeholders for attribute names, and a dictionary of - placeholders for attribute values. Here is a sample return value: - - ('#n0 = :v0', {'#n0': 'myattribute'}, {':v1': 'myvalue'}) - """ - if not isinstance(condition, ConditionBase): - raise DynamoDBNeedsConditionError(condition) - attribute_name_placeholders = {} - attribute_value_placeholders = {} - condition_expression = self._build_expression( - condition, - attribute_name_placeholders, - attribute_value_placeholders, - is_key_condition=is_key_condition, - ) - return BuiltConditionExpression( - condition_expression=condition_expression, - attribute_name_placeholders=attribute_name_placeholders, - attribute_value_placeholders=attribute_value_placeholders, - ) - - def _build_expression( - self, - condition, - attribute_name_placeholders, - attribute_value_placeholders, - is_key_condition, - ): - expression_dict = condition.get_expression() - replaced_values = [] - for value in expression_dict['values']: - # Build the necessary placeholders for that value. - # Placeholders are built for both attribute names and values. - replaced_value = self._build_expression_component( - value, - attribute_name_placeholders, - attribute_value_placeholders, - condition.has_grouped_values, - is_key_condition, - ) - replaced_values.append(replaced_value) - # Fill out the expression using the operator and the - # values that have been replaced with placeholders. - return expression_dict['format'].format( - *replaced_values, operator=expression_dict['operator'] - ) - - def _build_expression_component( - self, - value, - attribute_name_placeholders, - attribute_value_placeholders, - has_grouped_values, - is_key_condition, - ): - # Continue to recurse if the value is a ConditionBase in order - # to extract out all parts of the expression. - if isinstance(value, ConditionBase): - return self._build_expression( - value, - attribute_name_placeholders, - attribute_value_placeholders, - is_key_condition, - ) - # If it is not a ConditionBase, we can recurse no further. - # So we check if it is an attribute and add placeholders for - # its name - elif isinstance(value, AttributeBase): - if is_key_condition and not isinstance(value, Key): - raise DynamoDBNeedsKeyConditionError( - f'Attribute object {value.name} is of type {type(value)}. ' - f'KeyConditionExpression only supports Attribute objects ' - f'of type Key' - ) - return self._build_name_placeholder( - value, attribute_name_placeholders - ) - # If it is anything else, we treat it as a value and thus placeholders - # are needed for the value. - else: - return self._build_value_placeholder( - value, attribute_value_placeholders, has_grouped_values - ) - - def _build_name_placeholder(self, value, attribute_name_placeholders): - attribute_name = value.name - # Figure out which parts of the attribute name that needs replacement. - attribute_name_parts = ATTR_NAME_REGEX.findall(attribute_name) - - # Add a temporary placeholder for each of these parts. - placeholder_format = ATTR_NAME_REGEX.sub('%s', attribute_name) - str_format_args = [] - for part in attribute_name_parts: - name_placeholder = self._get_name_placeholder() - self._name_count += 1 - str_format_args.append(name_placeholder) - # Add the placeholder and value to dictionary of name placeholders. - attribute_name_placeholders[name_placeholder] = part - # Replace the temporary placeholders with the designated placeholders. - return placeholder_format % tuple(str_format_args) - - def _build_value_placeholder( - self, value, attribute_value_placeholders, has_grouped_values=False - ): - # If the values are grouped, we need to add a placeholder for - # each element inside of the actual value. - if has_grouped_values: - placeholder_list = [] - for v in value: - value_placeholder = self._get_value_placeholder() - self._value_count += 1 - placeholder_list.append(value_placeholder) - attribute_value_placeholders[value_placeholder] = v - # Assuming the values are grouped by parenthesis. - # IN is the currently the only one that uses this so it maybe - # needed to be changed in future. - return '(' + ', '.join(placeholder_list) + ')' - # Otherwise, treat the value as a single value that needs only - # one placeholder. - else: - value_placeholder = self._get_value_placeholder() - self._value_count += 1 - attribute_value_placeholders[value_placeholder] = value - return value_placeholder - - -# class InternalDBESDKDynamoDBConditionExpressionBuilder: -# """This class is used to build condition expressions with placeholders""" - -# def __init__(self): -# self._name_count = 0 -# self._value_count = 0 -# self._name_placeholder = "n" -# self._value_placeholder = "v" - -# def _get_name_placeholder(self): -# return "#" + self._name_placeholder + str(self._name_count) - -# def _get_value_placeholder(self): -# return ":" + self._value_placeholder + str(self._value_count) - -# def reset(self): -# """Resets the placeholder name and values""" -# self._name_count = 0 -# self._value_count = 0 - -# def build_expression( -# self, condition, attribute_name_placeholders, attribute_value_placeholders, is_key_condition=False -# ): -# """ -# Builds the condition expression and the dictionary of placeholders. - -# :type condition: ConditionBase -# :param condition: A condition to be built into a condition expression -# string with any necessary placeholders. - -# :type is_key_condition: Boolean -# :param is_key_condition: True if the expression is for a -# KeyConditionExpression. False otherwise. - -# :rtype: (string, dict, dict) -# :returns: Will return a string representing the condition with -# placeholders inserted where necessary, a dictionary of -# placeholders for attribute names, and a dictionary of -# placeholders for attribute values. Here is a sample return value: - -# ('#n0 = :v0', {'#n0': 'myattribute'}, {':v1': 'myvalue'}) -# """ -# if not isinstance(condition, ConditionBase): -# raise DynamoDBNeedsConditionError(condition) -# condition_expression = self._build_expression( -# condition, -# attribute_name_placeholders, -# attribute_value_placeholders, -# is_key_condition=is_key_condition, -# ) -# return BuiltConditionExpression( -# condition_expression=condition_expression, -# attribute_name_placeholders=attribute_name_placeholders, -# attribute_value_placeholders=attribute_value_placeholders, -# ) - -# def _build_expression( -# self, -# condition, -# attribute_name_placeholders, -# attribute_value_placeholders, -# is_key_condition, -# ): -# expression_dict = condition.get_expression() -# replaced_values = [] -# for value in expression_dict["values"]: -# # Build the necessary placeholders for that value. -# # Placeholders are built for both attribute names and values. -# replaced_value = self._build_expression_component( -# value, -# attribute_name_placeholders, -# attribute_value_placeholders, -# condition.has_grouped_values, -# is_key_condition, -# ) -# replaced_values.append(replaced_value) -# # Fill out the expression using the operator and the -# # values that have been replaced with placeholders. -# return expression_dict["format"].format(*replaced_values, operator=expression_dict["operator"]) - -# def _build_expression_component( -# self, -# value, -# attribute_name_placeholders, -# attribute_value_placeholders, -# has_grouped_values, -# is_key_condition, -# ): -# # Continue to recurse if the value is a ConditionBase in order -# # to extract out all parts of the expression. -# if isinstance(value, ConditionBase): -# return self._build_expression( -# value, -# attribute_name_placeholders, -# attribute_value_placeholders, -# is_key_condition, -# ) -# # If it is not a ConditionBase, we can recurse no further. -# # So we check if it is an attribute and add placeholders for -# # its name -# elif isinstance(value, AttributeBase): -# if is_key_condition and not isinstance(value, Key): -# raise DynamoDBNeedsKeyConditionError( -# f"Attribute object {value.name} is of type {type(value)}. " -# f"KeyConditionExpression only supports Attribute objects " -# f"of type Key" -# ) -# return self._build_name_placeholder(value, attribute_name_placeholders) -# # If it is anything else, we treat it as a value and thus placeholders -# # are needed for the value. -# else: -# return self._build_value_placeholder(value, attribute_value_placeholders, has_grouped_values) - -# def _build_name_placeholder(self, value, attribute_name_placeholders): -# attribute_name = value.name -# # Figure out which parts of the attribute name that needs replacement. -# attribute_name_parts = ATTR_NAME_REGEX.findall(attribute_name) - -# # Add a temporary placeholder for each of these parts. -# placeholder_format = ATTR_NAME_REGEX.sub("%s", attribute_name) -# str_format_args = [] -# for part in attribute_name_parts: -# # If the the name is already an AttributeName, use it. Don't make a new placeholder. -# if part in attribute_name_placeholders: -# str_format_args.append(part) -# else: -# name_placeholder = self._get_name_placeholder() -# self._name_count += 1 -# str_format_args.append(name_placeholder) -# # Add the placeholder and value to dictionary of name placeholders. -# attribute_name_placeholders[name_placeholder] = part -# # Replace the temporary placeholders with the designated placeholders. -# return placeholder_format % tuple(str_format_args) - -# def _build_value_placeholder(self, value, attribute_value_placeholders, has_grouped_values=False): -# # If the values are grouped, we need to add a placeholder for -# # each element inside of the actual value. - -# # Also, you can define a grouped value with a colon here. -# # If it's a colon, it's not a grouped value for the sake of this logic. -# # Treat it as an "else" case. -# if has_grouped_values: -# placeholder_list = [] -# # If it's a pre-defined grouped attribute, don't attempt to unpack it as if it were -# for v in value: -# # If the value is already an AttributeValue, reuse it. Don't make a new placeholder. -# if v in attribute_value_placeholders: -# placeholder_list.append(v) -# else: -# value_placeholder = self._get_value_placeholder() -# self._value_count += 1 -# placeholder_list.append(value_placeholder) -# attribute_value_placeholders[value_placeholder] = v -# # Assuming the values are grouped by parenthesis. -# # IN is the currently the only one that uses this so it maybe -# # needed to be changed in future. -# return "(" + ", ".join(placeholder_list) + ")" -# # Otherwise, treat the value as a single value that needs only -# # one placeholder. -# else: -# # If the value is already an AttributeValue, reuse it. Don't make a new placeholder. -# if value in attribute_value_placeholders: -# return value -# else: -# value_placeholder = self._get_value_placeholder() -# self._value_count += 1 -# attribute_value_placeholders[value_placeholder] = value -# return value_placeholder From b70496143e2c1671c28aa47012e48da166350bd4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 May 2025 14:07:30 -0700 Subject: [PATCH 5/9] sync --- .../runtimes/python/test/requests.py | 32 +++++++++++-------- .../runtimes/python/test/responses.py | 12 +++---- .../unit/internal/test_client_to_resource.py | 2 +- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/DynamoDbEncryption/runtimes/python/test/requests.py b/DynamoDbEncryption/runtimes/python/test/requests.py index 705b9ede1..bcbd0a0ca 100644 --- a/DynamoDbEncryption/runtimes/python/test/requests.py +++ b/DynamoDbEncryption/runtimes/python/test/requests.py @@ -10,6 +10,8 @@ ) # Base request structures that are shared between DDB and dict formats +# Use ConsistentRead: True for all requests; +# many of these are used in integ tests, where consistent reads reduce test flakiness. def base_put_item_request(item): @@ -19,7 +21,7 @@ def base_put_item_request(item): def base_get_item_request(item): """Base structure for get_item requests.""" - return {"Key": {"partition_key": item["partition_key"], "sort_key": item["sort_key"]}} + return {"Key": {"partition_key": item["partition_key"], "sort_key": item["sort_key"]}, "ConsistentRead": True} def base_delete_item_request(item): @@ -32,6 +34,7 @@ def base_query_request(item): return { "KeyConditionExpression": "partition_key = :pk", "ExpressionAttributeValues": {":pk": item["partition_key"]}, + "ConsistentRead": True, } @@ -40,6 +43,7 @@ def base_scan_request(item): return { "FilterExpression": "attribute2 = :a2", "ExpressionAttributeValues": {":a2": item["attribute2"]}, + "ConsistentRead": True, } @@ -50,7 +54,7 @@ def base_batch_write_item_request(actions_with_items): def base_batch_get_item_request(keys): """Base structure for batch_get_item requests.""" - return {"RequestItems": {INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME: {"Keys": keys}}} + return {"RequestItems": {INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME: {"Keys": keys, "ConsistentRead": True}}} def base_transact_write_item_request(actions_with_items): @@ -142,7 +146,7 @@ def basic_batch_execute_statement_request_plaintext_table(): # No exhaustive requests are intended to be able to be used as real requests. # Some parameters conflict with each other when sent to DynamoDB. -# These are only intended to test the conversion of the structure from client to resource format. +# These are only intended to test the conversion of the structure between client and resource formats. def base_exhaustive_put_item_request(item): @@ -150,7 +154,7 @@ def base_exhaustive_put_item_request(item): Base structure for exhaustive put_item requests. This is not intended to be able to be used as a real request. Some parameters conflict with each other when sent to DynamoDB. - This is only intended to test the conversion of the request from client to resource format. + This is only intended to test the conversion of the request between client and resource formats. """ return { # Expected is legacy, but still in the boto3 docs. @@ -174,7 +178,7 @@ def base_exhaustive_get_item_request(item): Base structure for exhaustive get_item requests. This is not intended to be able to be used as a real request. Some parameters conflict with each other when sent to DynamoDB. - This is only intended to test the conversion of the request from client to resource format. + This is only intended to test the conversion of the request between client and resource formats. """ return { "ReturnConsumedCapacity": "TOTAL", @@ -196,7 +200,7 @@ def base_exhaustive_delete_item_request(item): Base structure for exhaustive delete_item requests. This is not intended to be able to be used as a real request. Some parameters conflict with each other when sent to DynamoDB. - This is only intended to test the conversion of the request from client to resource format. + This is only intended to test the conversion of the request between client and resource formats. """ return { "ReturnConsumedCapacity": "TOTAL", @@ -211,7 +215,7 @@ def base_exhaustive_query_request(item): Base structure for exhaustive query requests. This is not intended to be able to be used as a real request. Some parameters conflict with each other when sent to DynamoDB. - This is only intended to test the conversion of the request from client to resource format. + This is only intended to test the conversion of the request between client and resource formats. """ return { "IndexName": "index_name", @@ -240,7 +244,7 @@ def base_exhaustive_scan_request(item): Base structure for exhaustive scan requests. This is not intended to be able to be used as a real request. Some parameters conflict with each other when sent to DynamoDB. - This is only intended to test the conversion of the request from client to resource format. + This is only intended to test the conversion of the request between client and resource formats. """ return { "IndexName": "index_name", @@ -310,7 +314,7 @@ def exhaustive_query_request_ddb(item): Query request with all possible parameters. This is not intended to be able to be used as a real request. Some parameters conflict with each other when sent to DynamoDB. - This is only intended to test the conversion of the request from client to resource format. + This is only intended to test the conversion of the request between client and resource formats. """ base = basic_query_request_ddb(item) additional_keys = base_exhaustive_query_request(item) @@ -390,6 +394,7 @@ def basic_query_paginator_request(key): "TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, "KeyConditionExpression": "partition_key = :pk AND sort_key = :sk", "ExpressionAttributeValues": {":pk": key["partition_key"], ":sk": key["sort_key"]}, + "ConsistentRead": True, } @@ -399,6 +404,7 @@ def basic_scan_paginator_request(item): "TableName": INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME, "FilterExpression": "partition_key = :pk AND sort_key = :sk", "ExpressionAttributeValues": {":pk": item["partition_key"], ":sk": item["sort_key"]}, + "ConsistentRead": True, } @@ -427,7 +433,7 @@ def exhaustive_put_item_request_dict(item): Get a put_item request in dict format for any item. This is not intended to be able to be used as a real request. Some parameters conflict with each other when sent to DynamoDB. - This is only intended to test the conversion of the request from client to resource format. + This is only intended to test the conversion of the request between client and resource formats. """ base = basic_put_item_request_dict(item) # Replace the default ConditionExpression string with a ConditionExpression object @@ -452,7 +458,7 @@ def exhaustive_get_item_request_dict(item): Get a get_item request in dict format for any item. This is not intended to be able to be used as a real request. Some parameters conflict with each other when sent to DynamoDB. - This is only intended to test the conversion of the request from client to resource format. + This is only intended to test the conversion of the request between client and resource formats. """ base = basic_get_item_request_dict(item) additional_keys = base_exhaustive_get_item_request(item) @@ -478,7 +484,7 @@ def exhaustive_query_request_dict(item): Get a query request in dict format for any item. This is not intended to be able to be used as a real request. Some parameters conflict with each other when sent to DynamoDB. - This is only intended to test the conversion of the request from client to resource format. + This is only intended to test the conversion of the request between client and resource formats. """ base = basic_query_request_dict(item) additional_keys = base_exhaustive_query_request(item) @@ -495,7 +501,7 @@ def exhaustive_scan_request_dict(item): Get a scan request in dict format for any item. This is not intended to be able to be used as a real request. Some parameters conflict with each other when sent to DynamoDB. - This is only intended to test the conversion of the request from client to resource format. + This is only intended to test the conversion of the request between client and resource formats. """ base = basic_scan_request_dict(item) additional_keys = base_exhaustive_scan_request(item) diff --git a/DynamoDbEncryption/runtimes/python/test/responses.py b/DynamoDbEncryption/runtimes/python/test/responses.py index 9c4ba2a18..a54c17448 100644 --- a/DynamoDbEncryption/runtimes/python/test/responses.py +++ b/DynamoDbEncryption/runtimes/python/test/responses.py @@ -13,7 +13,7 @@ def exhaustive_put_item_response(item): Get a put_item response in resource (ddb) format for any item. This is not intended to be a real response that DynamoDB would return, but the response should contain additional attributes that DynamoDB could return. - This is only intended to exhaustively test the conversion of the request from client to resource format. + This is only intended to exhaustively test the conversion of the request between client and resource formats. """ base = basic_put_item_response(item) additional_keys = { @@ -38,7 +38,7 @@ def exhaustive_get_item_response(item): Get a get_item response in resource (ddb) format for any item. This is not intended to be a real response that DynamoDB would return, but the response should contain additional attributes that DynamoDB could return. - This is only intended to exhaustively test the conversion of the request from client to resource format. + This is only intended to exhaustively test the conversion of the request between client and resource formats. """ base = basic_get_item_response(item) additional_keys = { @@ -62,7 +62,7 @@ def exhaustive_query_response(items): Get a query response in resource (ddb) format for any items. This is not intended to be a real response that DynamoDB would return, but the response should contain additional attributes that DynamoDB could return. - This is only intended to exhaustively test the conversion of the request from client to resource format. + This is only intended to exhaustively test the conversion of the request between client and resource formats. """ base = basic_query_response(items) additional_keys = { @@ -83,7 +83,7 @@ def exhaustive_scan_response(items, keys): Get a scan response in resource (ddb) format for any items. This is not intended to be a real response that DynamoDB would return, but the response should contain additional attributes that DynamoDB could return. - This is only intended to exhaustively test the conversion of the request from client to resource format. + This is only intended to exhaustively test the conversion of the request between client and resource formats. """ base = basic_scan_response(items, keys) additional_keys = { @@ -105,7 +105,7 @@ def exhaustive_batch_get_item_response(items): Get a batch_get_item response in resource (ddb) format for any items. This is not intended to be a real response that DynamoDB would return, but the response should contain additional attributes that DynamoDB could return. - This is only intended to exhaustively test the conversion of the request from client to resource format. + This is only intended to exhaustively test the conversion of the request between client and resource formats. """ base = basic_batch_get_item_response(items) additional_keys = { @@ -130,7 +130,7 @@ def exhaustive_batch_write_item_put_response(items): Get a batch_write_item response in resource (ddb) format for any items. This is not intended to be a real response that DynamoDB would return, but the response should contain additional attributes that DynamoDB could return. - This is only intended to exhaustively test the conversion of the request from client to resource format. + This is only intended to exhaustively test the conversion of the request between client and resource formats. """ base = basic_batch_write_item_put_response(items) additional_keys = { diff --git a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py index e3f7e2f02..85107cc65 100644 --- a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py +++ b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py @@ -1,9 +1,9 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 import pytest +from boto3.dynamodb.conditions import ConditionExpressionBuilder from aws_dbesdk_dynamodb.internal.client_to_resource import ClientShapeToResourceShapeConverter -from boto3.dynamodb.conditions import ConditionExpressionBuilder from ...items import ( complex_item_ddb, From 0f5647e0a7604cf99e6f3b5f2aa75a111d0a094e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 29 May 2025 13:21:56 -0700 Subject: [PATCH 6/9] sync --- .../python/test/unit/internal/test_resource_to_client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py index 4655ff914..90e40d790 100644 --- a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py +++ b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py @@ -476,8 +476,6 @@ def assert_condition_expressions_are_equal(expected_item, actual_item, key): actual_item["ExpressionAttributeNames"] if "ExpressionAttributeNames" in actual_item else {}, actual_item["ExpressionAttributeValues"] if "ExpressionAttributeValues" in actual_item else {}, ) - print(f"{expected_key_condition_expression=}") - print(f"{actual_key_condition_expression=}") assert expected_key_condition_expression == actual_key_condition_expression From d1e160a50e30f13bea99a7a6b49d8cd22049871d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 30 May 2025 11:51:57 -0700 Subject: [PATCH 7/9] sync --- .../runtimes/python/test/requests.py | 11 +++ .../runtimes/python/test/responses.py | 93 +++++++++++++++++++ .../python/test/unit/internal/README.md | 39 ++++++++ .../unit/internal/test_client_to_resource.py | 66 +++++++++---- .../unit/internal/test_resource_to_client.py | 89 +++++++++++++----- 5 files changed, 259 insertions(+), 39 deletions(-) create mode 100644 DynamoDbEncryption/runtimes/python/test/unit/internal/README.md diff --git a/DynamoDbEncryption/runtimes/python/test/requests.py b/DynamoDbEncryption/runtimes/python/test/requests.py index bcbd0a0ca..900af6cf4 100644 --- a/DynamoDbEncryption/runtimes/python/test/requests.py +++ b/DynamoDbEncryption/runtimes/python/test/requests.py @@ -260,6 +260,17 @@ def base_exhaustive_scan_request(item): } +# No exhaustive requests for: +# - transact_write_items +# - transact_get_items +# - batch_write_item +# - batch_get_item +# - batch_execute_statement +# - execute_statement +# - execute_transaction +# The base requests sufficiently test the conversion of the request between client and resource formats +# for items. + # DDB format request functions diff --git a/DynamoDbEncryption/runtimes/python/test/responses.py b/DynamoDbEncryption/runtimes/python/test/responses.py index a54c17448..b765a0c09 100644 --- a/DynamoDbEncryption/runtimes/python/test/responses.py +++ b/DynamoDbEncryption/runtimes/python/test/responses.py @@ -154,16 +154,109 @@ def basic_transact_write_items_response(items): } +# No exhaustive response for transact_write_items; +# The basic_transact_write_items_response is sufficient + + def basic_transact_get_items_response(items): """Get a transact_get_items response in resource (ddb) format for any items.""" return {"Responses": [{"Item": item} for item in items]} +# No exhaustive response for transact_get_items; +# The basic_transact_get_items_response is sufficient + + def basic_update_item_response(item): """Get an update_item response in resource (ddb) format for any item.""" return {"Attributes": item} +def exhaustive_update_item_response(item): + """ + Get an update_item response in resource (ddb) format for any item. + This is not intended to be a real response that DynamoDB would return, + but the response should contain additional attributes that DynamoDB could return. + This is only intended to exhaustively test the conversion of the request between client and resource formats. + """ + base = basic_update_item_response(item) + additional_keys = { + "ItemCollectionMetrics": { + "ItemCollectionKey": {"partition_key": item["partition_key"]}, + }, + } + return {**base, **additional_keys} + + def basic_delete_item_response(item): """Get a delete_item response in resource (ddb) format for any item.""" return {"Attributes": item} + + +def exhaustive_delete_item_response(item): + """ + Get a delete_item response in resource (ddb) format for any item. + This is not intended to be a real response that DynamoDB would return, + but the response should contain additional attributes that DynamoDB could return. + This is only intended to exhaustively test the conversion of the request between client and resource formats. + """ + base = basic_delete_item_response(item) + additional_keys = { + "ItemCollectionMetrics": { + "ItemCollectionKey": {"partition_key": item["partition_key"]}, + }, + } + return {**base, **additional_keys} + + +def basic_execute_statement_response(items): + """Get an execute_statement response in resource (ddb) format for any items.""" + return {"Items": items} + + +def exhaustive_execute_statement_response(items): + """ + Get an execute_statement response in resource (ddb) format for any items. + This is not intended to be a real response that DynamoDB would return, + but the response should contain additional attributes that DynamoDB could return. + This is only intended to exhaustively test the conversion of the request between client and resource formats. + """ + base = basic_execute_statement_response(items) + additional_keys = { + "LastEvaluatedKey": { + "partition_key": items[-1]["partition_key"], + "sort_key": items[-1]["sort_key"], + }, + } + return {**base, **additional_keys} + + +def basic_execute_transaction_response(items): + """Get an execute_transaction response in resource (ddb) format for any items.""" + return {"Responses": [{"Item": item} for item in items]} + + +# No exhaustive response for execute_transaction; +# The basic_execute_transaction_response is sufficient + + +def basic_batch_execute_statement_response(items): + """Get a batch_execute_statement response in resource (ddb) format for any items.""" + return {"Responses": [{"Item": item} for item in items]} + + +def exhaustive_batch_execute_statement_response(items): + """ + Get a batch_execute_statement response in resource (ddb) format for any items. + This is not intended to be a real response that DynamoDB would return, + but the response should contain additional attributes that DynamoDB could return. + This is only intended to exhaustively test the conversion of the request between client and resource formats. + """ + base = basic_batch_execute_statement_response(items) + base["Responses"][0]["Error"] = { + "Item": { + "partition_key": items[0]["partition_key"], + "sort_key": items[0]["sort_key"], + } + } + return base diff --git a/DynamoDbEncryption/runtimes/python/test/unit/internal/README.md b/DynamoDbEncryption/runtimes/python/test/unit/internal/README.md new file mode 100644 index 000000000..df5c3f8e0 --- /dev/null +++ b/DynamoDbEncryption/runtimes/python/test/unit/internal/README.md @@ -0,0 +1,39 @@ +The `test_client_to_resource.py` and `test_resource_to_client.py` files +in this directory verify that DBESDK's boto3 resource/client conversion methods +correctly convert between resource/client shapes for all operations +supported by DBESDK. + +The only shapes that require conversion are +* `AttributeValue`s (DDB items or keys) + * Client format example: `{"S": "some string"}` + * Resource format example: `"some string"` +* ConditionExpressions (`KeyConditionExpression` or `FilterExpression`; only resource-to-client) + * Client shape ex.: + * KeyConditionExpression: `"attr : :value"` + * ExpressionAttributeValues: `{":value" : {"S" : "some value}}` + * Resource shape ex.: + * KeyConditionExpression: `Attr("attr").eq("some value")` + * (Resources also support the client-style string expression) + +The conversion logic will recursively traverse inpuyt/output shapes to find shapes that require conversion, then convert them. +(ex. for boto3 Table [put_item](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/put_item.html)), +the following kwargs MUST be converted from resource to client format: +* `["Item"]` +* `["Expected"][]["Value"]` +* `["Expected"][]["AttributeValueList"]` +* `["ConditionExpression"]` +* `["ExpressionAttributeValues"]` + +The requests, responses, and items in the parent directory are shared between the integ tests and these unit tests. +The integ tests send the exact request that whose client/resource conversion is tested in the unit tests, +and the integ tests receive the exact response whose conversion is tested in the unit tests. + +The integration tests verify that the basic forms of these requests and responses are authoritative. The unit tests verify that DBESDK’s conversion logic exactly transforms one shape format into the other. + +Note: The conversion logic is generated by Smithy-Dafny Python +and the shape traversals are derived from the MPL's DynamoDB Smithy model. +As a result, the correctness of this conversion logic is primarily depends on the correctness of the Smithy codegen logic and the correctness of the DynamoDB Smithy model. + +Originally, the conversion logic was hand-written, +so these tests go beyond smoke testing to provide extra guarantees, +even though basic smoke testing should suffice now that the logic is machine-generated. diff --git a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py index 85107cc65..c040e5fac 100644 --- a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py +++ b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_client_to_resource.py @@ -55,9 +55,12 @@ exhaustive_scan_request_dict, ) from ...responses import ( + basic_batch_execute_statement_response, basic_batch_get_item_response, basic_batch_write_item_put_response, basic_delete_item_response, + basic_execute_statement_response, + basic_execute_transaction_response, basic_get_item_response, basic_put_item_response, basic_query_response, @@ -65,12 +68,16 @@ basic_transact_get_items_response, basic_transact_write_items_response, basic_update_item_response, + exhaustive_batch_execute_statement_response, exhaustive_batch_get_item_response, exhaustive_batch_write_item_put_response, + exhaustive_delete_item_response, + exhaustive_execute_statement_response, exhaustive_get_item_response, exhaustive_put_item_response, exhaustive_query_response, exhaustive_scan_response, + exhaustive_update_item_response, ) client_to_resource_converter = ClientShapeToResourceShapeConverter() @@ -560,7 +567,6 @@ def test_GIVEN_test_transact_get_items_response_WHEN_client_to_resource_THEN_ret def test_update_item_request_ddb(): # Select unsigned attribute without loss of generality; # resource/client logic doesn't care about signed attributes - # TODO: Add exhaustive request return basic_update_item_request_ddb_unsigned_attribute @@ -568,7 +574,6 @@ def test_update_item_request_ddb(): def test_update_item_request_dict(): # Select unsigned attribute without loss of generality; # resource/client logic doesn't care about signed attributes - # TODO: Add exhaustive request return basic_update_item_request_dict_unsigned_attribute @@ -584,8 +589,9 @@ def test_GIVEN_test_update_item_request_WHEN_client_to_resource_THEN_returns_dic @pytest.fixture -def test_update_item_response(): - # TODO: Add exhaustive response +def test_update_item_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_update_item_response return basic_update_item_response @@ -616,14 +622,22 @@ def test_GIVEN_test_execute_statement_request_WHEN_client_to_resource_THEN_retur assert dict_item == test_execute_statement_request(test_dict_item) -def test_GIVEN_test_execute_statement_response_WHEN_client_to_resource_THEN_raises_NotImplementedError(): +@pytest.fixture +def test_execute_statement_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_execute_statement_response + return basic_execute_statement_response + + +def test_GIVEN_test_execute_statement_response_WHEN_client_to_resource_THEN_returns_dict_value( + test_execute_statement_response, test_ddb_item, test_dict_item +): # Given: Execute statement response - # TODO: this - ddb_response = {} + ddb_response = test_execute_statement_response([test_ddb_item]) # When: Converting to resource format resource_response = client_to_resource_converter.execute_statement_response(ddb_response) # Then: Returns dict value - assert resource_response == {} + assert resource_response == test_execute_statement_response([test_dict_item]) @pytest.fixture @@ -642,18 +656,24 @@ def test_GIVEN_test_execute_transaction_request_WHEN_client_to_resource_THEN_ret assert dict_item == test_execute_transaction_request(test_dict_item) -def test_GIVEN_test_execute_transaction_response_WHEN_client_to_resource_THEN_returns_dict_value(): +@pytest.fixture +def test_execute_transaction_response(): + return basic_execute_transaction_response + + +def test_GIVEN_test_execute_transaction_response_WHEN_client_to_resource_THEN_returns_dict_value( + test_execute_transaction_response, test_ddb_item, test_dict_item +): # Given: Execute transaction response - # TODO: this - ddb_response = {} + ddb_response = test_execute_transaction_response([test_ddb_item]) # When: Converting to resource format resource_response = client_to_resource_converter.execute_transaction_response(ddb_response) # Then: Returns dict value - assert resource_response == {} + assert resource_response == test_execute_transaction_response([test_dict_item]) @pytest.fixture -def test_batch_execute_statement_request(): +def test_batch_execute_statement_request(use_exhaustive_request): return basic_batch_execute_statement_request_encrypted_table @@ -668,14 +688,22 @@ def test_GIVEN_test_batch_execute_statement_request_WHEN_client_to_resource_THEN assert dict_item == test_batch_execute_statement_request() -def test_GIVEN_test_batch_execute_statement_response_WHEN_client_to_resource_THEN_raises_NotImplementedError(): +@pytest.fixture +def test_batch_execute_statement_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_batch_execute_statement_response + return basic_batch_execute_statement_response + + +def test_GIVEN_test_batch_execute_statement_response_WHEN_client_to_resource_THEN_returns_dict_value( + test_batch_execute_statement_response, test_ddb_item, test_dict_item +): # Given: Batch execute statement response - # TODO: this - ddb_response = {} + ddb_response = test_batch_execute_statement_response([test_ddb_item]) # When: Converting to resource format resource_response = client_to_resource_converter.batch_execute_statement_response(ddb_response) # Then: Returns dict value - assert resource_response == {} + assert resource_response == test_batch_execute_statement_response([test_dict_item]) @pytest.fixture @@ -700,7 +728,9 @@ def test_GIVEN_test_delete_item_request_WHEN_client_to_resource_THEN_returns_dic @pytest.fixture -def test_delete_item_response(): +def test_delete_item_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_delete_item_response return basic_delete_item_response diff --git a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py index 90e40d790..c3973adb9 100644 --- a/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py +++ b/DynamoDbEncryption/runtimes/python/test/unit/internal/test_resource_to_client.py @@ -56,9 +56,12 @@ exhaustive_scan_request_dict, ) from ...responses import ( + basic_batch_execute_statement_response, basic_batch_get_item_response, basic_batch_write_item_put_response, basic_delete_item_response, + basic_execute_statement_response, + basic_execute_transaction_response, basic_get_item_response, basic_put_item_response, basic_query_response, @@ -66,12 +69,16 @@ basic_transact_get_items_response, basic_transact_write_items_response, basic_update_item_response, + exhaustive_batch_execute_statement_response, exhaustive_batch_get_item_response, exhaustive_batch_write_item_put_response, + exhaustive_delete_item_response, + exhaustive_execute_statement_response, exhaustive_get_item_response, exhaustive_put_item_response, exhaustive_query_response, exhaustive_scan_response, + exhaustive_update_item_response, ) resource_to_client_converter = ResourceShapeToClientShapeConverter(table_name=INTEG_TEST_DEFAULT_DYNAMODB_TABLE_NAME) @@ -881,8 +888,9 @@ def test_GIVEN_update_item_request_without_table_name_WHEN_resource_to_client_TH @pytest.fixture -def test_update_item_response(): - # TODO: Add exhaustive response +def test_update_item_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_update_item_response return basic_update_item_response @@ -891,7 +899,7 @@ def test_GIVEN_update_item_response_WHEN_resource_to_client_THEN_returns_dict_va ): # Given: Update item response response = test_update_item_response(test_dict_item) - # When: Converting to resource format + # When: Converting to client format actual_ddb_response = resource_to_client_converter.update_item_response(response) # Then: Returns dict value expected_ddb_response = test_update_item_response(test_ddb_item) @@ -912,20 +920,33 @@ def test_GIVEN_test_execute_statement_request_WHEN_resource_to_client_THEN_retur ): # Given: Execute statement request request = test_execute_statement_request(test_dict_item) - # When: Converting to resource format + # When: Converting to client format actual_ddb_request = resource_to_client_converter.execute_statement_request(request) # Then: Returns dict value (here, request is not modified) assert actual_ddb_request == test_execute_statement_request(test_ddb_item) -def test_GIVEN_test_execute_statement_response_WHEN_resource_to_client_THEN_returns_dict_value(): +@pytest.fixture +def test_execute_statement_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_execute_statement_response + return basic_execute_statement_response + + +def test_GIVEN_test_execute_statement_response_WHEN_resource_to_client_THEN_returns_dict_value( + test_execute_statement_response, test_ddb_item, test_dict_item +): # Given: Execute statement response - # TODO: this - dict_response = {} - # When: Converting to resource format - ddb_response = resource_to_client_converter.execute_statement_response(dict_response) + response = test_execute_statement_response([test_dict_item]) + # When: Converting to client format + actual_ddb_response = resource_to_client_converter.execute_statement_response(response) # Then: Returns dict value - assert ddb_response == {} + actual_ddb_response = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_response, "Items") + expected_ddb_response = sort_attribute_list_of_dynamodb_json_lists( + test_execute_statement_response([test_ddb_item]), "Items" + ) + + assert actual_ddb_response == expected_ddb_response @pytest.fixture @@ -944,14 +965,25 @@ def test_GIVEN_test_execute_transaction_request_WHEN_resource_to_client_THEN_ret assert actual_ddb_request == test_execute_transaction_request(test_ddb_item) -def test_GIVEN_test_execute_transaction_response_WHEN_resource_to_client_THEN_returns_dict_value(): +@pytest.fixture +def test_execute_transaction_response(): + return basic_execute_transaction_response + + +def test_GIVEN_test_execute_transaction_response_WHEN_resource_to_client_THEN_returns_dict_value( + test_execute_transaction_response, test_ddb_item, test_dict_item +): # Given: Execute transaction response - # TODO: this - dict_response = {} + response = test_execute_transaction_response([test_dict_item]) # When: Converting to resource format - ddb_response = resource_to_client_converter.execute_transaction_response(dict_response) + actual_ddb_response = resource_to_client_converter.execute_transaction_response(response) # Then: Returns dict value - assert ddb_response == {} + expected_ddb_response = test_execute_transaction_response([test_ddb_item]) + + actual_ddb_response = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_response, "Responses") + expected_ddb_response = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_response, "Responses") + + assert actual_ddb_response == expected_ddb_response @pytest.fixture @@ -970,14 +1002,27 @@ def test_GIVEN_test_batch_execute_statement_request_WHEN_resource_to_client_THEN assert actual_ddb_request == test_batch_execute_statement_request() -def test_GIVEN_test_batch_execute_statement_response_WHEN_resource_to_client_THEN_returns_dict_value(): +@pytest.fixture +def test_batch_execute_statement_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_batch_execute_statement_response + return basic_batch_execute_statement_response + + +def test_GIVEN_test_batch_execute_statement_response_WHEN_resource_to_client_THEN_returns_dict_value( + test_batch_execute_statement_response, test_ddb_item, test_dict_item +): # Given: Batch execute statement response - # TODO: this - dict_response = {} + response = test_batch_execute_statement_response([test_dict_item]) # When: Converting to resource format - ddb_response = resource_to_client_converter.batch_execute_statement_response(dict_response) + actual_ddb_response = resource_to_client_converter.batch_execute_statement_response(response) # Then: Returns dict value - assert ddb_response == {} + expected_ddb_response = test_batch_execute_statement_response([test_ddb_item]) + + actual_ddb_response = sort_attribute_list_of_dynamodb_json_lists(actual_ddb_response, "Responses") + expected_ddb_response = sort_attribute_list_of_dynamodb_json_lists(expected_ddb_response, "Responses") + + assert actual_ddb_response == expected_ddb_response @pytest.fixture @@ -1014,7 +1059,9 @@ def test_GIVEN_delete_item_request_without_table_name_WHEN_resource_to_client_TH @pytest.fixture -def test_delete_item_response(): +def test_delete_item_response(use_exhaustive_request): + if use_exhaustive_request: + return exhaustive_delete_item_response return basic_delete_item_response From 360baf289843857c8a17ae2b43602bbedf8b6459 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 30 May 2025 11:53:19 -0700 Subject: [PATCH 8/9] sync --- .../python/test/unit/internal/README.md | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/DynamoDbEncryption/runtimes/python/test/unit/internal/README.md b/DynamoDbEncryption/runtimes/python/test/unit/internal/README.md index df5c3f8e0..f9c33964e 100644 --- a/DynamoDbEncryption/runtimes/python/test/unit/internal/README.md +++ b/DynamoDbEncryption/runtimes/python/test/unit/internal/README.md @@ -4,25 +4,27 @@ correctly convert between resource/client shapes for all operations supported by DBESDK. The only shapes that require conversion are -* `AttributeValue`s (DDB items or keys) - * Client format example: `{"S": "some string"}` - * Resource format example: `"some string"` -* ConditionExpressions (`KeyConditionExpression` or `FilterExpression`; only resource-to-client) - * Client shape ex.: - * KeyConditionExpression: `"attr : :value"` - * ExpressionAttributeValues: `{":value" : {"S" : "some value}}` - * Resource shape ex.: - * KeyConditionExpression: `Attr("attr").eq("some value")` - * (Resources also support the client-style string expression) + +- `AttributeValue`s (DDB items or keys) + - Client format example: `{"S": "some string"}` + - Resource format example: `"some string"` +- ConditionExpressions (`KeyConditionExpression` or `FilterExpression`; only resource-to-client) + - Client shape ex.: + - KeyConditionExpression: `"attr : :value"` + - ExpressionAttributeValues: `{":value" : {"S" : "some value}}` + - Resource shape ex.: + - KeyConditionExpression: `Attr("attr").eq("some value")` + - (Resources also support the client-style string expression) The conversion logic will recursively traverse inpuyt/output shapes to find shapes that require conversion, then convert them. (ex. for boto3 Table [put_item](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb/table/put_item.html)), the following kwargs MUST be converted from resource to client format: -* `["Item"]` -* `["Expected"][]["Value"]` -* `["Expected"][]["AttributeValueList"]` -* `["ConditionExpression"]` -* `["ExpressionAttributeValues"]` + +- `["Item"]` +- `["Expected"][]["Value"]` +- `["Expected"][]["AttributeValueList"]` +- `["ConditionExpression"]` +- `["ExpressionAttributeValues"]` The requests, responses, and items in the parent directory are shared between the integ tests and these unit tests. The integ tests send the exact request that whose client/resource conversion is tested in the unit tests, From aabc4fa9a4c3db81eed0ee356329ae428d229ba0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 30 May 2025 11:54:18 -0700 Subject: [PATCH 9/9] sync --- .../runtimes/python/test/unit/internal/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DynamoDbEncryption/runtimes/python/test/unit/internal/README.md b/DynamoDbEncryption/runtimes/python/test/unit/internal/README.md index f9c33964e..10b60d584 100644 --- a/DynamoDbEncryption/runtimes/python/test/unit/internal/README.md +++ b/DynamoDbEncryption/runtimes/python/test/unit/internal/README.md @@ -21,8 +21,8 @@ The conversion logic will recursively traverse inpuyt/output shapes to find shap the following kwargs MUST be converted from resource to client format: - `["Item"]` -- `["Expected"][]["Value"]` -- `["Expected"][]["AttributeValueList"]` +- `["Expected"][]["Value"]` +- `["Expected"][]["AttributeValueList"]` - `["ConditionExpression"]` - `["ExpressionAttributeValues"]`