From 030aba49efac25fbacd5c412d3369622fe53eb5d Mon Sep 17 00:00:00 2001 From: irina-herciu Date: Fri, 23 May 2025 15:53:55 +0300 Subject: [PATCH 01/13] implement query with expressions - wip --- .../DynamoDBv2/Custom/DataModel/Configs.cs | 16 + .../DynamoDBv2/Custom/DataModel/Context.cs | 2 + .../Custom/DataModel/ContextExpression.cs | 101 ++ .../Custom/DataModel/ContextInternal.cs | 875 +++++++++++++++++- .../Custom/DataModel/QueryConfig.cs | 9 + .../Custom/DataModel/TransactWrite.cs | 2 +- .../Custom/DataModel/_async/Context.Async.cs | 22 + .../_async/IDynamoDBContext.Async.cs | 29 +- .../Custom/DataModel/_bcl/Context.Sync.cs | 20 + .../DataModel/_bcl/IDynamoDBContext.Sync.cs | 27 +- .../IntegrationTests/DataModelTests.cs | 487 +++++++++- .../DataModelOperationSpecificConfigTests.cs | 2 +- .../MockabilityTests/AsyncSearchTests.cs | 2 +- 13 files changed, 1555 insertions(+), 39 deletions(-) create mode 100644 sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs index 0f6bc30780d5..46d737f8cfe5 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs @@ -269,6 +269,14 @@ public class DynamoDBOperationConfig /// public List QueryFilter { get; set; } + /// + /// Represents a filter expression that can be used to filter results in DynamoDB operations. + /// + /// + /// Note: Conditions must be against non-key properties. + /// + public ContextExpression ExpressionFilter { get; set; } + /// /// Default constructor /// @@ -281,6 +289,14 @@ public DynamoDBOperationConfig() /// Checks if the IndexName is set on the config /// internal bool IsIndexOperation { get { return !string.IsNullOrEmpty(IndexName); } } + + internal void ValidateFilter() + { + if (QueryFilter is { Count: > 0 } && ExpressionFilter is { Filter: not null } ) + { + throw new InvalidOperationException("Cannot specify both QueryFilter and ExpressionFilter in the same operation configuration. Please use one or the other."); + } + } } /// diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/Context.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/Context.cs index 37e886672e3a..ea7cb810f071 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/Context.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/Context.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; using System.Threading; #if AWS_ASYNC_API using System.Threading.Tasks; @@ -56,6 +57,7 @@ public partial class DynamoDBContext : IDynamoDBContext #endregion #region Public methods + /// public void RegisterTableDefinition(Table table) { diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs new file mode 100644 index 000000000000..b86b8073e6db --- /dev/null +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs @@ -0,0 +1,101 @@ +using System; +using System.Linq.Expressions; +using Amazon.DynamoDBv2.DocumentModel; +using Expression = System.Linq.Expressions.Expression; + +namespace Amazon.DynamoDBv2.DataModel +{ + /// + /// Represents a context expression for DynamoDB operations in the object-persistence programming model. + /// + public class ContextExpression + { + /// + /// Represents a filter expression that can be used to filter results in DynamoDB operations. + /// + public Expression Filter { get; private set; } + + /// + /// Sets the filter expression for DynamoDB operations. + /// + /// + /// + /// + public void SetFilter(Expression> filterExpression) + { + if (filterExpression == null) + { + throw new ArgumentNullException(nameof(filterExpression), "Filter expression cannot be null."); + } + Filter = filterExpression.Body; + } + } + + /// + /// Extensions for LINQ operations in DynamoDB. + /// + public static class LinqDdbExtensions + { + /// + /// Checks if a value is between two other values, inclusive. + /// + /// This method is only used inside expression trees; it should never be called at runtime. + /// + /// + /// + /// + /// + /// + public static bool Between(this T value, T lower, T upper) => throw null!; + + /// + /// Checks if a value is not between two other values, inclusive. + /// + /// This method is only used inside expression trees; it should never be called at runtime. + /// + /// + /// + public static bool AttributeExists(this object _) => throw null!; + + /// + /// Checks if a value does not have a specific attribute. + /// + /// This method is only used inside expression trees; it should never be called at runtime. + /// + /// + /// + public static bool AttributeNotExists(this object _) => throw null!; + + /// + /// Checks if a value has a specific attribute type. + /// + /// This method is only used inside expression trees; it should never be called at runtime. + /// + /// + /// + /// + public static bool AttributeType(this object _, DynamoDBAttributeType dynamoDbType) => throw null!; + } + + /// + /// Represents a node in a path expression for DynamoDB operations. + /// + internal class PathNode + { + public string Path { get; } + + public string FormattedPath { get; } + + public int IndexDepth { get; } + + public bool IsMap { get; } + + public PathNode(string path, int indexDepth, bool isMap, string formattedPath) + { + Path = path; + IndexDepth = indexDepth; + IsMap = isMap; + FormattedPath = formattedPath; + } + } +} \ No newline at end of file diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs index 806354cc9a68..dffca5ae491b 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs @@ -26,7 +26,9 @@ using Amazon.Util.Internal; using System.Globalization; using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; using ThirdParty.RuntimeBackports; +using Expression = System.Linq.Expressions.Expression; namespace Amazon.DynamoDBv2.DataModel { @@ -88,9 +90,9 @@ private static Document CreateExpectedDocumentForVersion(ItemStorage storage) return document; } - internal static Expression CreateConditionExpressionForVersion(ItemStorage storage, DynamoDBEntry.AttributeConversionConfig conversionConfig) + internal static DocumentModel.Expression CreateConditionExpressionForVersion(ItemStorage storage, DynamoDBEntry.AttributeConversionConfig conversionConfig) { - if (!storage.Config.HasVersion) return new Expression(); + if (!storage.Config.HasVersion) return new DocumentModel.Expression(); bool shouldExist = storage.CurrentVersion?.ConvertToExpectedAttributeValue(conversionConfig).Exists ?? false; string variableName = Common.GetVariableName("version"); @@ -99,7 +101,7 @@ internal static Expression CreateConditionExpressionForVersion(ItemStorage stora if (!shouldExist) { - return new Expression + return new DocumentModel.Expression { ExpressionStatement = $"attribute_not_exists({attributeReference})", ExpressionAttributeNames = { [attributeReference] = versionAttributeName } @@ -107,7 +109,7 @@ internal static Expression CreateConditionExpressionForVersion(ItemStorage stora } string attributeValueReference = Common.GetAttributeValueReference(variableName); - return new Expression + return new DocumentModel.Expression { ExpressionStatement = $"{attributeReference} = {attributeValueReference}", ExpressionAttributeNames = { [attributeReference] = versionAttributeName }, @@ -793,6 +795,7 @@ private bool TryToList(object value, [DynamicallyAccessedMembers(DynamicallyAcce } return true; } + private bool TryToScalar(object value, Type type, DynamoDBFlatConfig flatConfig, ref DynamoDBEntry entry) { var elementType = Utils.GetElementType(type); @@ -964,16 +967,20 @@ private ScanFilter ComposeScanFilter(IEnumerable conditions, Item private QueryFilter ComposeQueryFilter(DynamoDBFlatConfig currentConfig, object hashKeyValue, IEnumerable conditions, ItemStorageConfig storageConfig, out List indexNames) { - if (hashKeyValue == null) - throw new ArgumentNullException("hashKeyValue"); + ValidateHashKey(hashKeyValue, storageConfig); + var hashKeyEntry = HashKeyValueToDynamoDBEntry(currentConfig, hashKeyValue, storageConfig); - if (storageConfig.HashKeyPropertyNames == null || storageConfig.HashKeyPropertyNames.Count == 0) + Document hashKey = new Document { - throw new InvalidOperationException($"Attempted to make a query without a defined hash key attribute. " + - $"If using {nameof(DynamoDBContextConfig.DisableFetchingTableMetadata)}, ensure that the table's hash key " + - $"is annotated with {nameof(DynamoDBHashKeyAttribute)}."); - } + [hashKeyEntry.Item1] = hashKeyEntry.Item2 + }; + return ComposeQueryFilterHelper(currentConfig, hashKey, conditions, storageConfig, out indexNames); + } + + private (string,DynamoDBEntry) HashKeyValueToDynamoDBEntry(DynamoDBFlatConfig currentConfig, object hashKeyValue, + ItemStorageConfig storageConfig) + { // Set hash key property name // In case of index queries, if GSI, different key could be used string hashKeyProperty = storageConfig.HashKeyPropertyNames[0]; @@ -985,26 +992,26 @@ private QueryFilter ComposeQueryFilter(DynamoDBFlatConfig currentConfig, object DynamoDBEntry hashKeyEntry = ValueToDynamoDBEntry(propertyStorage, hashKeyValue, currentConfig); if (hashKeyEntry == null) throw new InvalidOperationException("Unable to convert hash key value for property " + hashKeyProperty); - Document hashKey = new Document(); - hashKey[hashAttributeName] = hashKeyEntry; + return (hashAttributeName,hashKeyEntry); + } - return ComposeQueryFilterHelper(currentConfig, hashKey, conditions, storageConfig, out indexNames); + private static void ValidateHashKey(object hashKeyValue, ItemStorageConfig storageConfig) + { + if (hashKeyValue == null) + throw new ArgumentNullException("hashKeyValue"); + + if (storageConfig.HashKeyPropertyNames == null || storageConfig.HashKeyPropertyNames.Count == 0) + { + throw new InvalidOperationException($"Attempted to make a query without a defined hash key attribute. " + + $"If using {nameof(DynamoDBContextConfig.DisableFetchingTableMetadata)}, ensure that the table's hash key " + + $"is annotated with {nameof(DynamoDBHashKeyAttribute)}."); + } } private static string NO_INDEX = DynamoDBFlatConfig.DefaultIndexName; - // This method composes the query filter and determines the possible indexes that the filter - // may be used against. In the case where the condition property is also a RANGE key on the - // table and not just on LSI/GSI, the potential index will be "" (absent). - private QueryFilter ComposeQueryFilterHelper( - DynamoDBFlatConfig currentConfig, - Document hashKey, - IEnumerable conditions, - ItemStorageConfig storageConfig, - out List indexNames) - { - if (hashKey == null) - throw new ArgumentNullException("hashKey"); + private void ValidateQueryKeyConfiguration(ItemStorageConfig storageConfig, DynamoDBFlatConfig currentConfig) + { if (storageConfig.HashKeyPropertyNames.Count != 1) { var tableName = GetTableName(storageConfig.TableName, currentConfig); @@ -1016,7 +1023,23 @@ private QueryFilter ComposeQueryFilterHelper( var tableName = GetTableName(storageConfig.TableName, currentConfig); throw new InvalidOperationException("Must have one range key or a GSI index defined for the table " + tableName); } + } + // This method composes the query filter and determines the possible indexes that the filter + // may be used against. In the case where the condition property is also a RANGE key on the + // table and not just on LSI/GSI, the potential index will be "" (absent). + private QueryFilter ComposeQueryFilterHelper( + DynamoDBFlatConfig currentConfig, + Document hashKey, + IEnumerable conditions, + ItemStorageConfig storageConfig, + out List indexNames) + { + if (hashKey == null) + throw new ArgumentNullException("hashKey"); + + ValidateQueryKeyConfiguration(storageConfig, currentConfig); + QueryFilter filter = new QueryFilter(); // Configure hash-key equality condition @@ -1131,6 +1154,607 @@ private static List CreateQueryConditions(DynamoDBFlatConfig fla return conditions; } + private DocumentModel.Expression ComposeExpression(Expression filterExpression, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) + { + DocumentModel.Expression filter = new DocumentModel.Expression(); + if (filterExpression == null) return filter; + + + var aliasList = new KeyAttributeAliasList(); + var expressionNode = BuildExpressionNode(filterExpression, storageConfig, flatConfig); + + filter.ExpressionStatement = expressionNode.BuildExpressionString(aliasList, "C"); + if (aliasList.NamesList != null && aliasList.NamesList.Count != 0) + { + var namesDictionary = new Dictionary(); + for (int i = 0; i < aliasList.NamesList.Count; i++) + { + namesDictionary[$"#C{i}"] = aliasList.NamesList[i]; + } + + filter.ExpressionAttributeNames = namesDictionary; + } + + if (aliasList.ValuesList != null && aliasList.ValuesList.Count != 0) + { + var values = new Dictionary(); + for (int i = 0; i < aliasList.ValuesList.Count; i++) + { + values[$":C{i}"] = aliasList.ValuesList[i]; + } + + filter.ExpressionAttributeValues = values; + } + + return filter; + } + + private ExpressionNode BuildExpressionNode(Expression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) + { + var node = new ExpressionNode(); + + switch (expr) + { + case LambdaExpression lambda: + // Recursively process the body of the lambda + return BuildExpressionNode(lambda.Body, storageConfig, flatConfig); + case BinaryExpression binary when IsComparison(binary.NodeType): + node = HandleBinaryComparison(binary, storageConfig, flatConfig); + break; + + case BinaryExpression binary: + // Handle AND/OR expressions + var left = BuildExpressionNode(binary.Left, storageConfig, flatConfig); + var right = BuildExpressionNode(binary.Right, storageConfig, flatConfig); + node.Children.Enqueue(left); + node.Children.Enqueue(right); + var condition = binary.NodeType == ExpressionType.AndAlso ? "AND" : "OR"; + node.FormatedExpression = $"(#c) {condition} (#c)"; + break; + + case MethodCallExpression method: + node = HandleMethodCall(method, storageConfig, flatConfig); + break; + + case UnaryExpression { NodeType: ExpressionType.Not } unary: + var notUnary = BuildExpressionNode(unary.Operand, storageConfig, flatConfig); + node.Children.Enqueue(notUnary); + node.FormatedExpression = $"NOT (#c)"; + break; + + default: + throw new InvalidOperationException($"Unsupported expression type: {expr.NodeType}"); + } + + return node; + } + + private ExpressionNode HandleBinaryComparison(BinaryExpression expr, ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) + { + Expression member = null; + ConstantExpression constant = null; + + if (IsMember(expr.Left)) + { + member = expr.Left; + constant = GetConstant(expr.Right); + } + else if (IsMember(expr.Right)) + { + member = expr.Right; + constant = GetConstant(expr.Left); + } + + if (member == null) + throw new NotSupportedException("Expected member access"); + + var node = new ExpressionNode + { + FormatedExpression = expr.NodeType switch + { + ExpressionType.Equal => "#c = #c", + ExpressionType.NotEqual => "#c <> #c", + ExpressionType.LessThan => "#c < #c", + ExpressionType.LessThanOrEqual => "#c <= #c", + ExpressionType.GreaterThan => "#c > #c", + ExpressionType.GreaterThanOrEqual => "#c >= #c", + _ => throw new InvalidOperationException($"Unsupported mode: {expr.NodeType}") + } + }; + + SetExpressionNodeAttributes(storageConfig, member, constant, node, flatConfig); + + return node; + } + + private ExpressionNode HandleMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) + { + // Handle method calls like Equals, Between, In, AttributeExists, AttributeNotExists, AttributeType, BeginsWith, Contains + return expr.Method.Name switch + { + "Equals" => HandleEqualsMethodCall(expr, storageConfig, flatConfig), + "Contains" => HandleContainsMethodCall(expr, storageConfig, flatConfig), + "StartsWith" => HandleStartsWithMethodCall(expr, storageConfig, flatConfig), + "In" => HandleInMethodCall(expr, storageConfig, flatConfig), + "Between" => HandleBetweenMethodCall(expr, storageConfig, flatConfig), + "AttributeExists" => HandleExistsMethodCall(expr, storageConfig, flatConfig), + "IsNull" or "AttributeNotExists" => HandleIsNullMethodCall(expr, storageConfig, flatConfig), + "AttributeType" => HandleAttributeTypeMethodCall(expr, storageConfig, flatConfig), + _ => throw new NotSupportedException($"Unsupported method call: {expr.Method.Name}") + }; + } + + private ExpressionNode HandleAttributeTypeMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) + { + var node = new ExpressionNode + { + FormatedExpression = "attribute_type (#c, #c)" + }; + + if (expr.Arguments.Count == 2 && expr.Object == null) + { + if (expr.Arguments[0] is MemberExpression memberObj && + expr.Arguments[1] is ConstantExpression typeExpr) + { + SetExpressionNodeAttributes(storageConfig, memberObj, typeExpr, node, flatConfig); + } + else + { + throw new NotSupportedException("Expected MemberExpression and ConstantExpression as arguments for AttributeType method call."); + } + } + else + { + throw new NotSupportedException("Expected MemberExpression and ConstantExpression as arguments for AttributeType method call."); + } + return node; + } + + private ExpressionNode HandleIsNullMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) + { + var node = new ExpressionNode { + FormatedExpression = "attribute_not_exists (#c)" + }; + + if (expr.Arguments.Count == 1 && expr.Object == null) + { + var collectionExpr = expr.Arguments[0] as MemberExpression; + if (collectionExpr != null) + { + SetExpressionNameNode(storageConfig, collectionExpr, node, flatConfig); + } + else + { + throw new NotSupportedException("Expected MemberExpression as argument for AttributeNotExists method call."); + } + } + else + { + throw new NotSupportedException("Expected MemberExpression as argument for AttributeNotExists method call."); + } + + return node; + } + + private ExpressionNode HandleExistsMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) + { + var node = new ExpressionNode + { + FormatedExpression = "attribute_exists (#c)" + }; + + if (expr.Arguments.Count == 1 && expr.Object == null) + { + var collectionExpr = expr.Arguments[0] as MemberExpression; + if (collectionExpr != null) + { + SetExpressionNameNode(storageConfig, collectionExpr, node, flatConfig); + } + else + { + throw new NotSupportedException("Expected MemberExpression as argument for AttributeExists method call."); + } + } + + return node; + } + + private ExpressionNode HandleInMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) + { + var node = new ExpressionNode + { + FormatedExpression = "#c IN (" + }; + + if (expr.Object is MemberExpression memberObj && expr.Arguments[0] is NewArrayExpression arrayExpr) + { + var propertyStorage = SetExpressionNameNode(storageConfig, memberObj, node, flatConfig); + + foreach (var arg in arrayExpr.Expressions) + { + if (arg is not ConstantExpression constExpr) continue; + + node.FormatedExpression += "#c, "; + + SetExpressionValueNode(constExpr, node, propertyStorage, flatConfig); + } + } + else + { + throw new NotSupportedException("Expected MemberExpression with NewArrayExpression as argument for In method call."); + } + + if (node.FormatedExpression.EndsWith(", ")) + { + node.FormatedExpression = node.FormatedExpression.Substring(0, node.FormatedExpression.Length - 2); + } + node.FormatedExpression += ")"; + return node; + } + + private ExpressionNode HandleBetweenMethodCall(MethodCallExpression expr, + ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) + { + var node = new ExpressionNode + { + FormatedExpression = "#c BETWEEN #c AND #c" + }; + + + if (expr.Arguments.Count == 3 && expr.Object == null) + { + var collectionExpr = expr.Arguments[0] as MemberExpression; + var constExprLeft = expr.Arguments[1] as ConstantExpression; + var constExprRight = expr.Arguments[2] as ConstantExpression; + + if (collectionExpr != null && constExprLeft != null && constExprRight != null) + { + var propertyStorage = SetExpressionNameNode(storageConfig, collectionExpr, node, flatConfig); + SetExpressionValueNode(constExprLeft, node, propertyStorage, flatConfig); + SetExpressionValueNode(constExprRight, node, propertyStorage, flatConfig); + } + } + else + { + throw new NotSupportedException("Expected MemberExpression with NewArrayExpression as argument for In method call."); + } + + return node; + } + + private ExpressionNode HandleStartsWithMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) + { + var node = new ExpressionNode + { + FormatedExpression = "begins_with (#c, #c)" + }; + if (expr.Object is MemberExpression memberObj && expr.Arguments[0] is ConstantExpression argConst) + { + SetExpressionNodeAttributes(storageConfig, memberObj, argConst, node,flatConfig); + } + else + { + throw new NotSupportedException("Expected MemberExpression with ConstantExpression as argument for StartsWith method call."); + } + + return node; + } + + private ExpressionNode HandleContainsMethodCall(MethodCallExpression expr, + ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) + { + var node = new ExpressionNode + { + FormatedExpression = "contains (#c, #c)" + }; + if (expr.Object is MemberExpression memberObj && expr.Arguments[0] is ConstantExpression argConst) + { + SetExpressionNodeAttributes(storageConfig, memberObj, argConst, node,flatConfig); + } + else if (expr.Arguments.Count == 2 && expr.Object == null) + { + var collectionExpr = expr.Arguments[0] as MemberExpression; + var constExpr = expr.Arguments[1] as ConstantExpression; + + if (collectionExpr != null && constExpr != null) + { + SetExpressionNodeAttributes(storageConfig, collectionExpr, constExpr, node,flatConfig); + } + else + { + throw new NotSupportedException( + "Expected MemberExpression with ConstantExpression as argument for Contains method call."); + } + } + else + { + throw new NotSupportedException( + "Expected MemberExpression with ConstantExpression as argument for Contains method call."); + } + + return node; + } + + private ExpressionNode HandleEqualsMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) + { + var node = new ExpressionNode + { + FormatedExpression = $"#c = #c" + }; + + if (expr.Object is MemberExpression member && + expr.Arguments[0] is ConstantExpression constant && + constant.Value == null) + { + SetExpressionNodeAttributes(storageConfig, member, constant, node, flatConfig); + return node; + } + else if (expr.Arguments.Count == 2 && expr.Object == null) + { + var memberObj = GetMember(expr.Arguments[0]) ?? GetMember(expr.Arguments[1]); + var argConst = GetConstant(expr.Arguments[1]) ?? GetConstant(expr.Arguments[0]); + if (memberObj != null && argConst != null) + { + SetExpressionNodeAttributes(storageConfig, memberObj, argConst, node, flatConfig); + return node; + } + } + + throw new NotSupportedException("Expected MemberExpression with ConstantExpression as argument for Equals method call."); + } + + private void SetExpressionNodeAttributes(ItemStorageConfig storageConfig, Expression memberObj, + ConstantExpression argConst, ExpressionNode node, DynamoDBFlatConfig flatConfig) + { + var propertyStorage = SetExpressionNameNode(storageConfig, memberObj, node, flatConfig); + SetExpressionValueNode(argConst, node, propertyStorage, flatConfig); + } + + private void SetExpressionValueNode(ConstantExpression argConst, ExpressionNode node, PropertyStorage propertyStorage, DynamoDBFlatConfig flatConfig) + { + DynamoDBEntry entry=ToDynamoDBEntry(propertyStorage, argConst?.Value, flatConfig, canReturnScalarInsteadOfList: true); + var valuesNode = new ExpressionNode() + { + FormatedExpression = $"#v" + }; + valuesNode.Values.Enqueue(entry); + node.Children.Enqueue(valuesNode); + } + + private PropertyStorage ResolveNestedPropertyStorage(StorageConfig rootConfig, DynamoDBFlatConfig flatConfig, + List path, Queue namesNodeNames) + { + StorageConfig currentConfig = rootConfig; + PropertyStorage propertyStorage= null; + for (int i = 0; i < path.Count; i++) + { + var pathNode = path[i]; + + // If the path node is a map, just add the name to the queue + if (pathNode.IsMap) + { + namesNodeNames.Enqueue(pathNode.Path); + continue; + } + + propertyStorage = currentConfig.GetPropertyStorage(pathNode.Path); + if (propertyStorage == null) + throw new InvalidOperationException($"Property '{pathNode.Path}' not found in storage config."); + // If the property is ignored, throw an exception + if (propertyStorage.IsIgnored) + { + throw new InvalidOperationException($"Property '{pathNode.Path}' is marked as ignored and cannot be used in a filter expression."); + } + + namesNodeNames.Enqueue(propertyStorage.AttributeName); + // If not the last segment, descend into the nested StorageConfig + if (i >= path.Count - 1) continue; + + // Only descend if the property is a complex type (not primitive/string) + var propertyType = propertyStorage.MemberType; + if (Utils.IsPrimitive(propertyType)) + throw new InvalidOperationException($"Property '{pathNode.Path}' is not a complex type."); + + // Determine the element type if the property is a collection + var nextPathNode = path[i + 1]; + + Type elementType = null; + var depth = pathNode.IndexDepth; + if (nextPathNode is { IsMap: true }) + { + depth += nextPathNode.IndexDepth; + } + + var nodePropertyType = propertyType; + var currentDepth = 0; + + while (currentDepth <= depth && nodePropertyType != null && Utils.ImplementsInterface(nodePropertyType, typeof(ICollection<>)) + && nodePropertyType != typeof(string)) + { + elementType = Utils.GetElementType(nodePropertyType); + if (elementType == null) + { + IsSupportedDictionaryType(nodePropertyType, out elementType); + } + nodePropertyType = elementType; + currentDepth++; + } + elementType ??= propertyType; + + ItemStorageConfig config = StorageConfigCache.GetConfig(elementType, flatConfig); + currentConfig = config.BaseTypeStorageConfig; + } + + return propertyStorage; + } + + private PropertyStorage SetExpressionNameNode(ItemStorageConfig storageConfig, Expression memberObj, + ExpressionNode node, DynamoDBFlatConfig flatConfig) + { + var path = ExtractPathNodes(memberObj); + if(path.Count == 0) + { + throw new InvalidOperationException("Expected a valid property path in the expression."); + } + var namesNode = new ExpressionNode() + { + FormatedExpression = string.Join(".", path.Select(pn => pn.FormattedPath)) + }; + + var propertyStorage = ResolveNestedPropertyStorage(storageConfig.BaseTypeStorageConfig, flatConfig, path, namesNode.Names); + node.Children.Enqueue(namesNode); + + return propertyStorage; + + List ExtractPathNodes(Expression expr) + { + var pathNodes = new List(); + int indexDepth = 0; + string indexed = string.Empty; + + while (expr != null) + { + switch (expr) + { + case MemberExpression memberExpr: + pathNodes.Insert(0, new PathNode(memberExpr.Member.Name, indexDepth, false, $"#n{indexed}")); + indexed = string.Empty; + indexDepth = 0; + expr = memberExpr.Expression; + break; + case MethodCallExpression methodCall + when methodCall.Method.Name == "First" || methodCall.Method.Name == "FirstOrDefault": + expr = methodCall.Arguments.Count > 0 ? methodCall.Arguments[0] : methodCall.Object; + indexDepth++; + indexed += "[0]"; + break; + case MethodCallExpression methodCall + when methodCall.Method.Name == "get_Item": + { + var arg = methodCall.Arguments[0]; + if (arg is ConstantExpression constArg) + { + var indexValue = constArg.Value; + switch (indexValue) + { + case int intValue: + indexDepth++; + indexed += $"[{intValue}]"; + break; + case string stringValue: + pathNodes.Insert(0, new PathNode(stringValue, indexDepth, true, $"#n{indexed}")); + indexDepth = 0; + indexed = string.Empty; + break; + default: + throw new NotSupportedException( + $"Indexer argument must be an integer or string, got {indexValue.GetType().Name}."); + } + } + else + { + throw new NotSupportedException( + $"Method {methodCall.Method.Name} is not supported in property path."); + } + + expr = methodCall.Object; + break; + } + case MethodCallExpression methodCall: + throw new NotSupportedException( + $"Method {methodCall.Method.Name} is not supported in property path."); + case UnaryExpression unaryExpr + when unaryExpr.NodeType == ExpressionType.Convert || unaryExpr.NodeType == ExpressionType.ConvertChecked: + // Handle conversion expressions (e.g., (int)someEnum) + expr = unaryExpr.Operand; + break; + + default: + expr = null; + break; + } + } + + return pathNodes; + } + } + + private static bool IsComparison(ExpressionType type) + { + return type is ExpressionType.Equal or ExpressionType.NotEqual or + ExpressionType.GreaterThan or ExpressionType.GreaterThanOrEqual or + ExpressionType.LessThan or ExpressionType.LessThanOrEqual; + } + + private static MemberExpression GetMember(Expression expr) + { + if (expr is MemberExpression memberExpr) + return memberExpr; + + if (expr is UnaryExpression ue) + return GetMember(ue.Operand); + + // Handle indexer access (get_Item) for lists/arrays/dictionaries + if (expr is MethodCallExpression methodCall && methodCall.Method.Name == "get_Item") + return GetMember(methodCall.Object); + + return null; + } + + private static bool IsMember(Expression expr) + { + if (expr is MemberExpression memberExpr) + return true; + + if (expr is UnaryExpression ue) + return IsMember(ue.Operand); + + return false; + } + + + private static ConstantExpression GetConstant(Expression expr) + { + var constant = expr as ConstantExpression; + if (constant != null) + return constant; + // If the expression is a UnaryExpression, check its Operand + var unary = expr as UnaryExpression; + if (unary != null) + { + return unary.Operand as ConstantExpression; + } + var newexp= expr as NewExpression; + if (newexp != null) + { + throw new NotSupportedException($"Unsupported expression type {expr.Type}"); + } + return null; + } + + private static DynamoDBEntry ToAttributeValue(object value) + { + //todo - add support for other types + return value switch + { + string s => s, + int i => i, + long l => l, + double d => d, + bool b => b, + _ => throw new NotSupportedException($"Unsupported value type: {value.GetType().Name}") + }; + } + // Key creation private DynamoDBEntry ValueToDynamoDBEntry(PropertyStorage propertyStorage, object value, DynamoDBFlatConfig flatConfig) { @@ -1277,6 +1901,33 @@ public ContextSearch(Search search, DynamoDBFlatConfig flatConfig) return new ContextSearch(scan, flatConfig); } + + private ContextSearch ConvertScan(ContextExpression filterExpression, DynamoDBOperationConfig operationConfig) + { + DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, this.Config); + ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); + + DocumentModel.Expression expression = null; + if (filterExpression is { Filter: null }) + { + expression = ComposeExpression(filterExpression.Filter, storageConfig, flatConfig); + } + + Table table = GetTargetTable(storageConfig, flatConfig); + var scanConfig = new ScanOperationConfig + { + AttributesToGet = storageConfig.AttributesToGet, + Select = SelectValues.SpecificAttributes, + FilterExpression = expression, + IndexName = flatConfig.IndexName, + ConsistentRead = flatConfig.ConsistentRead.GetValueOrDefault(false) + }; + + // table.Scan() returns the ISearch interface but we explicitly cast it to a Search object since we rely on its internal behavior + Search scan = table.Scan(scanConfig) as Search; + return new ContextSearch(scan, flatConfig); + } + private ContextSearch ConvertFromScan<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ScanOperationConfig scanConfig, DynamoDBOperationConfig operationConfig) { DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); @@ -1301,24 +1952,183 @@ public ContextSearch(Search search, DynamoDBFlatConfig flatConfig) private ContextSearch ConvertQueryByValue<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(object hashKeyValue, QueryOperator op, IEnumerable values, DynamoDBOperationConfig operationConfig) { + if (operationConfig!=null) + { + operationConfig.ValidateFilter(); + } + DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); - List conditions = CreateQueryConditions(flatConfig, op, values, storageConfig); - ContextSearch query = ConvertQueryByValue(hashKeyValue, conditions, operationConfig, storageConfig); + //todo - add support for expression + ContextSearch query; + if (operationConfig is { ExpressionFilter: { Filter: not null } }) + { + query = ConvertQueryByValueWithExpression(hashKeyValue, op, values, operationConfig.ExpressionFilter.Filter, operationConfig, storageConfig); + } + else + { + List conditions = CreateQueryConditions(flatConfig, op, values, storageConfig); + query = ConvertQueryByValue(hashKeyValue, conditions, operationConfig, storageConfig); + } return query; } - private ContextSearch ConvertQueryByValue<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(object hashKeyValue, IEnumerable conditions, DynamoDBOperationConfig operationConfig, ItemStorageConfig storageConfig = null) + private ContextSearch ConvertQueryByValueWithExpression(object hashKeyValue, QueryOperator op, IEnumerable values, + Expression filterExpression, DynamoDBOperationConfig operationConfig, ItemStorageConfig storageConfig) { DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); + if (storageConfig == null) storageConfig = StorageConfigCache.GetConfig(flatConfig); + if (operationConfig.QueryFilter != null && operationConfig.QueryFilter.Count != 0) + { + throw new InvalidOperationException("QueryFilter is not supported with filter expression. Use either QueryFilter or filter expression, but not both."); + } + return ConvertQueryHelper(hashKeyValue, op, values, flatConfig, storageConfig, filterExpression); - List indexNames; - QueryFilter filter = ComposeQueryFilter(flatConfig, hashKeyValue, conditions, storageConfig, out indexNames); - return ConvertQueryHelper(flatConfig, storageConfig, filter, indexNames); } + private ContextSearch + ConvertQueryByValue<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>( + object hashKeyValue, IEnumerable conditions, DynamoDBOperationConfig operationConfig, + ItemStorageConfig storageConfig = null) + { + // DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); + // if (storageConfig == null) + // storageConfig = StorageConfigCache.GetConfig(flatConfig); + + // List indexNames; + // QueryFilter filter = ComposeQueryFilter(flatConfig, hashKeyValue, conditions, storageConfig, out indexNames); + // return ConvertQueryHelper(flatConfig, storageConfig, filter, indexNames); + + if (operationConfig != null) + { + operationConfig.ValidateFilter(); + } + + DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); + + if (storageConfig == null) + storageConfig = StorageConfigCache.GetConfig(flatConfig); + + ContextSearch query; + if (operationConfig is { ExpressionFilter: { Filter: not null } }) + { + query = ConvertQueryByValueWithExpression(hashKeyValue, QueryOperator.Equal, null, + operationConfig.ExpressionFilter.Filter, operationConfig, storageConfig); + } + else + { + + List indexNames; + QueryFilter filter = ComposeQueryFilter(flatConfig, hashKeyValue, conditions, storageConfig, out indexNames); + query = ConvertQueryHelper(flatConfig, storageConfig, filter, indexNames); + } + + return query; + } + + private ContextSearch + ConvertQueryHelper<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>( + object hashKeyValue, QueryOperator op, IEnumerable values, DynamoDBFlatConfig flatConfig, + ItemStorageConfig storageConfig, + Expression filterExpression) + { + ValidateHashKey(hashKeyValue, storageConfig); + ValidateQueryKeyConfiguration(storageConfig, flatConfig); + + var hashKeyEntry = HashKeyValueToDynamoDBEntry(flatConfig, hashKeyValue, storageConfig); + var keyExpression = new DocumentModel.Expression + { + ExpressionStatement = "#hashKey = :hashKey", + ExpressionAttributeValues = new Dictionary + { + { ":hashKey", hashKeyEntry.Item2 } + }, + ExpressionAttributeNames = new Dictionary + { + { "#hashKey", hashKeyEntry.Item1 } + } + }; + + string rangeKeyPropertyName; + + string indexName = flatConfig.IndexName; + if (string.IsNullOrEmpty(indexName)) + rangeKeyPropertyName = storageConfig.RangeKeyPropertyNames.FirstOrDefault(); + else + rangeKeyPropertyName = storageConfig.GetRangeKeyByIndex(indexName); + + if (!string.IsNullOrEmpty(rangeKeyPropertyName) && values!=null) + { + //todo implement QueryOperator to expression mapping + keyExpression.ExpressionStatement += GetRangeKeyConditionExpression($"#rangeKey", op); + keyExpression.ExpressionAttributeNames.Add("#rangeKey", rangeKeyPropertyName); + var valuesList = values?.ToList(); + if (op == QueryOperator.Between && valuesList != null && valuesList.Count() == 2) + { + keyExpression.ExpressionAttributeValues.Add(":rangeKey0", ToAttributeValue(valuesList.ElementAt(0))); + keyExpression.ExpressionAttributeValues.Add(":rangeKey1", ToAttributeValue(valuesList.ElementAt(1))); + } + else + { + keyExpression.ExpressionAttributeValues.Add(":rangeKey0", ToAttributeValue(valuesList.FirstOrDefault())); + } + } + + Table table = GetTargetTable(storageConfig, flatConfig); + var queryConfig = new QueryOperationConfig + { + ConsistentRead = flatConfig.ConsistentRead.Value, + BackwardSearch = flatConfig.BackwardQuery.Value, + KeyExpression = keyExpression, + }; + + var expression = ComposeExpression(filterExpression, storageConfig, flatConfig); + + //TODO string indexName = GetQueryIndexName(currentConfig, indexNames); + queryConfig.FilterExpression = expression; + + if (string.IsNullOrEmpty(indexName)) + { + queryConfig.Select = SelectValues.SpecificAttributes; + List attributesToGet = storageConfig.AttributesToGet; + queryConfig.AttributesToGet = attributesToGet; + } + else + { + queryConfig.IndexName = indexName; + queryConfig.Select = SelectValues.AllProjectedAttributes; + } + Search query = table.Query(queryConfig) as Search; + + return new ContextSearch(query, flatConfig); + } + + private static string GetRangeKeyConditionExpression( string rangeKeyAlias, QueryOperator op) + { + switch (op) + { + case QueryOperator.Equal: + return $" AND {rangeKeyAlias} = :rangeKey0"; + case QueryOperator.LessThan: + return $" AND {rangeKeyAlias} < :rangeKey0"; + case QueryOperator.LessThanOrEqual: + return $" AND {rangeKeyAlias} <= :rangeKey0"; + case QueryOperator.GreaterThan: + return $" AND {rangeKeyAlias} > :rangeKey0"; + case QueryOperator.GreaterThanOrEqual: + return $" AND {rangeKeyAlias} >= :rangeKey0"; + case QueryOperator.Between: + return $" AND {rangeKeyAlias} BETWEEN :rangeKey0 AND :rangeKey0"; + case QueryOperator.BeginsWith: + return $" AND begins_with({rangeKeyAlias}, :rangeKey0)"; + default: + throw new NotSupportedException($"QueryOperator '{op}' is not supported for key conditions."); + } + } + + private ContextSearch ConvertQueryHelper(DynamoDBFlatConfig currentConfig, ItemStorageConfig storageConfig, QueryFilter filter, List indexNames) { Table table = GetTargetTable(storageConfig, currentConfig); @@ -1354,5 +2164,6 @@ private ContextSearch ConvertQueryHelper(DynamoDBFlatConfig currentConfig, It } #endregion + } } diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/QueryConfig.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/QueryConfig.cs index 46d28c5566ce..4999f76fe819 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/QueryConfig.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/QueryConfig.cs @@ -64,6 +64,14 @@ public class QueryConfig : BaseOperationConfig /// public List QueryFilter { get; set; } + /// + /// Represents a filter expression that can be used to filter results in DynamoDB operations. + /// + /// + /// Note: Expression filters must be against non-key properties. + /// + public ContextExpression ExpressionFilter { get; set; } + /// /// Property that directs to use consistent reads. /// If property is not set, behavior defaults to non-consistent reads. @@ -91,6 +99,7 @@ internal override DynamoDBOperationConfig ToDynamoDBOperationConfig() config.IndexName = IndexName; config.ConditionalOperator = ConditionalOperator; config.QueryFilter = QueryFilter; + config.ExpressionFilter = ExpressionFilter; config.ConsistentRead = ConsistentRead; config.RetrieveDateTimeInUtc = RetrieveDateTimeInUtc; diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/TransactWrite.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/TransactWrite.cs index 95a9f39a90d9..8e713783b813 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/TransactWrite.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/TransactWrite.cs @@ -424,7 +424,7 @@ private void CheckUseVersioning() } } - private Expression CreateConditionExpressionForVersion(ItemStorage storage) + private DocumentModel.Expression CreateConditionExpressionForVersion(ItemStorage storage) { if (!ShouldUseVersioning()) return null; var conversionConfig = new DynamoDBEntry.AttributeConversionConfig( diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/_async/Context.Async.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/_async/Context.Async.cs index 20e834aa2097..84861df63c98 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/_async/Context.Async.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/_async/Context.Async.cs @@ -21,6 +21,7 @@ using System.Threading; using System.Threading.Tasks; using Amazon.DynamoDBv2.DocumentModel; +using System.Linq.Expressions; namespace Amazon.DynamoDBv2.DataModel { @@ -343,6 +344,16 @@ public async Task ExecuteBatchGetAsync(params IBatchGet[] batches) } } + /// + public IAsyncSearch ScanAsync<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ContextExpression filterExpression) + { + using (DynamoDBTelemetry.CreateSpan(this, nameof(ScanAsync))) + { + var scan = ConvertScan(filterExpression, null); + return FromSearchAsync(scan); + } + } + /// [Obsolete("Use the ScanAsync overload that takes ScanConfig instead, since DynamoDBOperationConfig contains properties that are not applicable to ScanAsync.")] public IAsyncSearch ScanAsync<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(IEnumerable conditions, DynamoDBOperationConfig operationConfig = null) @@ -364,6 +375,17 @@ public async Task ExecuteBatchGetAsync(params IBatchGet[] batches) } } + /// + public IAsyncSearch ScanAsync<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ContextExpression filterExpression, ScanConfig scanConfig) + { + using (DynamoDBTelemetry.CreateSpan(this, nameof(ScanAsync))) + { + var scan = ConvertScan(filterExpression, scanConfig?.ToDynamoDBOperationConfig()); + return FromSearchAsync(scan); + } + } + + /// public IAsyncSearch FromScanAsync<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ScanOperationConfig scanConfig) { diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/_async/IDynamoDBContext.Async.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/_async/IDynamoDBContext.Async.cs index b2f25ee4e703..674b7f358a66 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/_async/IDynamoDBContext.Async.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/_async/IDynamoDBContext.Async.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq.Expressions; using System.Threading; using System.Threading.Tasks; using Amazon.DynamoDBv2.DocumentModel; @@ -499,6 +500,17 @@ partial interface IDynamoDBContext /// AsyncSearch which can be used to retrieve DynamoDB data. IAsyncSearch ScanAsync<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(IEnumerable conditions); + /// + /// Configures an async Scan operation against DynamoDB, finding items + /// that match the specified filter expression. + /// + /// Type of object. + /// + /// A LINQ expression used to filter the results. The expression is translated to a DynamoDB filter expression and applied server-side. + /// + /// AsyncSearch which can be used to retrieve DynamoDB data. + IAsyncSearch ScanAsync<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ContextExpression filterExpression); + /// /// Configures an async Scan operation against DynamoDB, finding items /// that match the specified conditions. @@ -525,6 +537,19 @@ partial interface IDynamoDBContext /// AsyncSearch which can be used to retrieve DynamoDB data. IAsyncSearch ScanAsync<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(IEnumerable conditions, ScanConfig scanConfig); + /// + /// Configures an async Scan operation against DynamoDB, finding items + /// that match the specified conditions. + /// + /// Type of object. + /// + /// A LINQ expression used to filter the results. The expression is translated to a DynamoDB filter expression and applied server-side. + /// + /// Config object that can be used to override properties on the table's context for this request. + /// AsyncSearch which can be used to retrieve DynamoDB data. + IAsyncSearch ScanAsync<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ContextExpression filterExpression, ScanConfig scanConfig); + + /// /// Configures an async Scan operation against DynamoDB, finding items /// that match the specified conditions. @@ -631,7 +656,7 @@ partial interface IDynamoDBContext /// /// Value(s) of the condition. /// For all operations except QueryOperator.Between, values should be one value. - /// For QueryOperator.Betwee, values should be two values. + /// For QueryOperator.Between, values should be two values. /// /// Config object that can be used to override properties on the table's context for this request. /// AsyncSearch which can be used to retrieve DynamoDB data. @@ -639,7 +664,7 @@ partial interface IDynamoDBContext /// /// Configures an async Query operation against DynamoDB using a mid-level document model - /// query configration, finding items that match the specified conditions. + /// query configuration, finding items that match the specified conditions. /// /// Type of object. /// Mid-level, document model query request object. diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/_bcl/Context.Sync.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/_bcl/Context.Sync.cs index a3d3892053b7..8582f9252f0f 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/_bcl/Context.Sync.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/_bcl/Context.Sync.cs @@ -297,6 +297,16 @@ public IEnumerable Scan(params ScanCondition[] conditions) } } + /// + public IEnumerable Scan(ContextExpression filterExpression) + { + using (DynamoDBTelemetry.CreateSpan(this, nameof(Scan))) + { + var scan = ConvertScan(filterExpression, null); + return FromSearch(scan); + } + } + /// [Obsolete("Use the Scan overload that takes ScanConfig instead, since DynamoDBOperationConfig contains properties that are not applicable to Scan.")] public IEnumerable Scan(IEnumerable conditions, DynamoDBOperationConfig operationConfig) @@ -318,6 +328,16 @@ public IEnumerable Scan(IEnumerable conditions, ScanConfig } } + /// + public IEnumerable Scan(ContextExpression filterExpression, ScanConfig scanConfig) + { + using (DynamoDBTelemetry.CreateSpan(this, nameof(Scan))) + { + var scan = ConvertScan(filterExpression, scanConfig?.ToDynamoDBOperationConfig()); + return FromSearch(scan); + } + } + /// public IEnumerable FromScan(ScanOperationConfig scanConfig) { diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/_bcl/IDynamoDBContext.Sync.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/_bcl/IDynamoDBContext.Sync.cs index e500bcb20ce8..2885c272f088 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/_bcl/IDynamoDBContext.Sync.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/_bcl/IDynamoDBContext.Sync.cs @@ -15,7 +15,7 @@ using System; using System.Collections.Generic; - +using System.Linq.Expressions; using Amazon.DynamoDBv2.DocumentModel; namespace Amazon.DynamoDBv2.DataModel @@ -413,6 +413,19 @@ partial interface IDynamoDBContext /// Lazy-loaded collection of results. IEnumerable Scan(params ScanCondition[] conditions); + /// + /// Executes a Scan operation against DynamoDB, + /// returning items that match the specified filter expression. + /// + /// Type of object. + /// + /// A LINQ expression used to filter the results. The expression is translated to a DynamoDB filter expression and applied server-side. + /// + /// + /// A lazy-loaded collection of results of type that match the filter expression. + /// + IEnumerable Scan(ContextExpression filterExpression); + /// /// Executes a Scan operation against DynamoDB, finding items /// that match the specified conditions. @@ -439,6 +452,18 @@ partial interface IDynamoDBContext /// Lazy-loaded collection of results. IEnumerable Scan(IEnumerable conditions, ScanConfig scanConfig); + /// + /// Executes a Scan operation against DynamoDB, finding items + /// that match the specified filter expression. + /// + /// Type of object. + /// + /// A LINQ expression used to filter the results. The expression is translated to a DynamoDB filter expression and applied server-side. + /// + /// Config object that can be used to override properties on the table's context for this request. + /// Lazy-loaded collection of results. + IEnumerable Scan(ContextExpression filterExpression, ScanConfig scanConfig); + /// /// Executes a Scan operation against DynamoDB, finding items /// that match the specified conditions. diff --git a/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs b/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs index 4d3e85ffa066..86fd1688f987 100644 --- a/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs +++ b/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs @@ -11,6 +11,9 @@ using Amazon.DynamoDBv2.DocumentModel; using Amazon.DynamoDBv2.DataModel; using System.Threading.Tasks; +using System.Collections.ObjectModel; +using Amazon.S3.Model; +using Amazon.Runtime.Internal.Transform; namespace AWSSDK_DotNet.IntegrationTests.Tests.DynamoDB @@ -269,7 +272,7 @@ public void TestTransactWrite_AddSaveItem_DocumentTransaction() /// /// Tests that the DynamoDB operations can retrieve attributes in UTC and local timezone. - /// + /// [TestMethod] [TestCategory("DynamoDBv2")] [DataRow(true)] @@ -556,6 +559,488 @@ public void TestContext_RetrieveDateTimeInUtc_OperationConfig(bool retrieveDateT Assert.AreEqual(employee.Age, storedEmployee.Age); } + [TestMethod] + [TestCategory("DynamoDBv2")] + public void TestContext_ScanWithExpression_NestedPaths() + { + TableCache.Clear(); + CleanupTables(); + TableCache.Clear(); + + var product1 = new Product + { + Id = 1, + Name = "Widget", + CompanyInfo = new CompanyInfo + { + Name = "Acme", + Founded = new DateTime(2000, 1, 1), + AllProducts = new List + { + new Product { Id = 2, Name = "Gadget" } + }, + FeaturedBrands = new[] { "Acme", "Contoso" } + }, + Price = 100 + }; + + var product2 = new Product + { + Id = 3, + Name = "Thing", + CompanyInfo = new CompanyInfo + { + Name = "Contoso", + Founded = new DateTime(2010, 5, 5), + AllProducts = new List + { + new Product { Id = 4, Name = "Device" } + }, + FeaturedBrands = new[] { "Contoso" } + }, + Price = 200 + }; + + var product3 = new Product + { + Id = 5, + Name = "CloudSpotter", + CompanyInfo = new CompanyInfo + { + Name = "Contoso", + Founded = new DateTime(2010, 5, 5), + AllProducts = new List + { + new Product + { + Id = 6, Name = "Service", Components = new List + { + "Code", + "Storage", + "Network" + } + } + }, + CompetitorProducts = new Dictionary>() + { + { + "CloudsAreOK", new List() + { + new Product() + { + Id = 8, Name = "CloudSpotter RipOff" + } + } + } + }, + FeaturedBrands = new[] { "Contoso" } + }, + Price = 200 + }; + + Context.Save(product1); + Context.Save(product2); + Context.Save(product3); + + // 1. Filter on a nested property (CompanyInfo.Name) + var expr1 = new ContextExpression(); + expr1.SetFilter(p => p.CompanyInfo.Name == "Acme"); + var byCompanyName = Context.Scan(expr1).ToList(); + Assert.AreEqual(1, byCompanyName.Count); + Assert.AreEqual("Widget", byCompanyName[0].Name); + + // 2. Filter on a nested array property (FeaturedBrands contains "Acme") + var expr2 = new ContextExpression(); + expr2.SetFilter(p => p.CompanyInfo.FeaturedBrands.Contains("Acme")); + var byFeaturedBrand = Context.Scan(expr2).ToList(); + Assert.AreEqual(1, byFeaturedBrand.Count); + Assert.AreEqual("Widget", byFeaturedBrand[0].Name); + + // 3. Filter on a double-nested property + var expr3 = new ContextExpression(); + expr3.SetFilter(p => p.CompanyInfo.AllProducts.First().Name == "Device"); + var byDoubleNested = Context.Scan(expr3).ToList(); + Assert.AreEqual(1, byDoubleNested.Count); + Assert.AreEqual("Thing", byDoubleNested[0].Name); + + var expr4 = new ContextExpression(); + expr4.SetFilter(p => p.CompanyInfo.AllProducts[0].Name == "Device"); + var byDoubleNested1 = Context.Scan(expr4).ToList(); + Assert.AreEqual(1, byDoubleNested1.Count); + Assert.AreEqual("Thing", byDoubleNested1[0].Name); + + // 4. Filter on a value inside a dictionary of lists + var expr5 = new ContextExpression(); + expr5.SetFilter(p => p.CompanyInfo.CompetitorProducts["CloudsAreOK"][0].Name == "CloudSpotter RipOff"); + var byDictionaryNested = Context.Scan(expr5).ToList(); + Assert.AreEqual(1, byDictionaryNested.Count); + Assert.AreEqual("CloudSpotter", byDictionaryNested[0].Name); + } + + + [TestMethod] + [TestCategory("DynamoDBv2")] + public void TestContext_Scan_WithExpressionFilter() + { + TableCache.Clear(); + CleanupTables(); + TableCache.Clear(); + + var employee = new Employee() + { + Name = "Bob", + Age = 45, + CurrentStatus = Status.Active, + CompanyName = "test", + }; + + var employee3 = new Employee + { + Name = "Cob", + Age = 45, + CurrentStatus = Status.Inactive, + CompanyName = "test1", + }; + + var employee2 = new Employee + { + Name = "Rob", + Age = 35, + CurrentStatus = Status.Active, + CompanyName = "test", + }; + + var employee4 = new Employee + { + Name = "Sam", + Age = 20, + CurrentStatus = Status.Upcoming, + CompanyName = "test2", + }; + + Context.Save(employee); + Context.Save(employee2); + Context.Save(employee3); + Context.Save(employee4); + + // Numeric equality + var exprAgeEq = new ContextExpression(); + exprAgeEq.SetFilter(e => e.Age == 45); + var ageEqResult = Context.Scan(exprAgeEq).ToList(); + Assert.AreEqual(2, ageEqResult.Count); + + var exprAgeEqM = new ContextExpression(); + exprAgeEqM.SetFilter(e => Equals(e.Age, 45)); + var ageEqMResult = Context.Scan(exprAgeEqM).ToList(); + Assert.AreEqual(2, ageEqMResult.Count); + + // AND expression with BinaryComparisons + var exprAnd = new ContextExpression(); + exprAnd.SetFilter(e => e.Age > 40 && e.CompanyName == "test"); + var andResults = Context.Scan(exprAnd).ToList(); + + var s1 = Context.Scan(new List() + { + new ScanCondition("Age", ScanOperator.GreaterThan, 40), + new ScanCondition("CompanyName", ScanOperator.Equal, "test") + }, new ScanConfig { RetrieveDateTimeInUtc = true }).ToList(); + + Assert.IsNotNull(s1); + Assert.AreEqual(s1.Count, 1); + Assert.AreEqual(s1.FirstOrDefault().Name, "Bob"); + + Assert.IsNotNull(andResults); + Assert.AreEqual(andResults.Count, 1); + Assert.AreEqual(andResults.FirstOrDefault().Name, "Bob"); + + // NOT expression + var exprNot = new ContextExpression(); + exprNot.SetFilter(e => !(e.CompanyName == "test1")); + var notResult = Context.Scan(exprNot).ToList(); + Assert.AreEqual(3, notResult.Count); + Assert.IsTrue(notResult.All(e => e.CompanyName != "test1")); + + // OR expression + var exprOr = new ContextExpression(); + exprOr.SetFilter(e => e.Name == "Bob" || e.Name == "Rob"); + var orResult = Context.Scan(exprOr).ToList(); + Assert.AreEqual(2, orResult.Count); + Assert.IsTrue(orResult.Any(e => e.Name == "Bob")); + Assert.IsTrue(orResult.Any(e => e.Name == "Rob")); + + // Contains on list property (Aliases) + var empWithAliases = new Employee + { + Name = "Ali", + Age = 50, + CurrentStatus = Status.Active, + MiddleName = "MiddleName", + CompanyName = "test", + Aliases = new List { "Al", "A", "B" } + }; + Context.Save(empWithAliases); + + var exprContains = new ContextExpression(); + exprContains.SetFilter(e => e.Aliases.Contains("Al")); + var containsResult = Context.Scan(exprContains).ToList(); + Assert.IsTrue(containsResult.Any(e => e.Name == "Ali")); + + var exprContainsEnumerable = new ContextExpression(); + exprContainsEnumerable.SetFilter(e => Enumerable.Contains(e.Aliases, "Al")); + var containsEnumerableResult = Context.Scan(exprContainsEnumerable).ToList(); + Assert.IsTrue(containsEnumerableResult.Any(e => e.Name == "Ali")); + + // String.StartsWith + var exprStartsWith = new ContextExpression(); + exprStartsWith.SetFilter(e => e.Name.StartsWith("B")); + var startsWithResult = Context.Scan(exprStartsWith).ToList(); + Assert.IsTrue(startsWithResult.Any(e => e.Name == "Bob")); + + // Between + var exprBetween = new ContextExpression(); + exprBetween.SetFilter(e => e.Age.Between(40, 50)); + var betweenResult = Context.Scan(exprBetween).ToList(); + Assert.AreEqual(3, betweenResult.Count); + Assert.IsTrue(betweenResult.All(e => e.Age >= 40 && e.Age <= 50)); + + // String.Contains + var exprStringContains = new ContextExpression(); + exprStringContains.SetFilter(e => e.Name.Contains("o")); + var stringContainsResult = Context.Scan(exprStringContains).ToList(); + Assert.IsTrue(stringContainsResult.Any(e => e.Name == "Bob" || e.Name == "Rob" || e.Name == "Cob")); + + var exprNullCheck = new ContextExpression(); + exprNullCheck.SetFilter(e => e.MiddleName.AttributeExists()); + var nullCheckResult = Context.Scan(exprNullCheck).ToList(); + Assert.IsTrue(nullCheckResult.Count == 1); + + var exprNull = new ContextExpression(); + exprNull.SetFilter(e => e.MiddleName.AttributeNotExists()); + var nullResult = Context.Scan(exprNull).ToList(); + Assert.IsTrue(nullResult.Count == 4); + + // --- Enum scenario --- + // Scan for employees with CurrentStatus == Status.Active + var exprActiveEnum = new ContextExpression(); + exprActiveEnum.SetFilter(e => e.CurrentStatus == Status.Active); + var activeEnumResult = Context.Scan(exprActiveEnum).ToList(); + Assert.AreEqual(3, activeEnumResult.Count); + Assert.IsTrue(activeEnumResult.All(e => e.CurrentStatus == Status.Active)); + + // Scan for employees with CurrentStatus == Status.Upcoming + var exprUpcomingEnum = new ContextExpression(); + exprUpcomingEnum.SetFilter(e => e.CurrentStatus == Status.Upcoming); + var upcomingEnumResult = Context.Scan(exprUpcomingEnum).ToList(); + Assert.AreEqual(1, upcomingEnumResult.Count); + Assert.AreEqual("Sam", upcomingEnumResult[0].Name); + } + + + [TestMethod] + [TestCategory("DynamoDBv2")] + public void TestContext_Query_WithExpressionFilter() + { + TableCache.Clear(); + CleanupTables(); + TableCache.Clear(); + + // Seed data + var employee1 = new Employee + { + Name = "Alice", + Age = 30, + CompanyName = "Contoso", + CurrentStatus = Status.Active + }; + var employee11 = new Employee + { + Name = "Alice", + Age = 35, + CompanyName = "ContosoTest", + CurrentStatus = Status.Active + }; + var employee2 = new Employee + { + Name = "Bob", + Age = 40, + CompanyName = "Acme", + CurrentStatus = Status.Inactive + }; + var employee3 = new Employee + { + Name = "Charlie", + Age = 35, + CompanyName = "Contoso", + CurrentStatus = Status.Active + }; + + Context.Save(employee1); + Context.Save(employee2); + Context.Save(employee3); + + var contextExpression = new ContextExpression(); + contextExpression.SetFilter(e => e.CompanyName == "Contoso"); + + var employees = Context.Query( + "Alice", + new QueryConfig + { + ExpressionFilter = contextExpression + }).ToList(); + + Assert.AreEqual(1, employees.Count); + Assert.AreEqual("Alice", employees[0].Name); + + employees = Context.Query( + "Charlie", + new QueryConfig + { + ExpressionFilter = contextExpression + }).ToList(); + + Assert.AreEqual(1, employees.Count); + Assert.AreEqual("Charlie", employees[0].Name); + + employees = Context.Query( + "Bob", + new QueryConfig + { + ExpressionFilter = contextExpression + }).ToList(); + + Assert.AreEqual(0, employees.Count); + } + + [TestMethod] + [TestCategory("DynamoDBv2")] + public void TestContext_Query_QueryFilter_vs_ExpressionFilter() + { + TableCache.Clear(); + CleanupTables(); + TableCache.Clear(); + + // Seed data + var employee1 = new Employee + { + Name = "Diane", + Age = 40, + CompanyName = "Big River", + CurrentStatus = Status.Active, + Score = 140, + ManagerName = "Eva" + }; + var employee2 = new Employee + { + Name = "Diane", + Age = 24, + CompanyName = "Big River", + CurrentStatus = Status.Inactive, + Score = 101, + ManagerName = "Eva" + }; + var employee3 = new Employee + { + Name = "Diane", + Age = 31, + CompanyName = "Small River", + CurrentStatus = Status.Active, + Score = 120, + ManagerName = "Barbara" + }; + Context.Save(employee1); + Context.Save(employee2); + Context.Save(employee3); + + // 1. QueryFilter only: filter by ManagerName == "Eva" + var queryFilter = new List + { + new ScanCondition("ManagerName", ScanOperator.Equal, "Eva") + }; + var resultQueryFilter = Context.Query("Diane", new QueryConfig + { + QueryFilter = queryFilter + }).ToList(); + + // 2. ExpressionFilter only: filter by ManagerName == "Eva" + var contextExpression = new ContextExpression(); + contextExpression.SetFilter(e => e.ManagerName == "Eva"); + var resultExpressionFilter = Context.Query("Diane", new QueryConfig + { + ExpressionFilter = contextExpression + }).ToList(); + + // Assert both results are equivalent + Assert.AreEqual(resultQueryFilter.Count, resultExpressionFilter.Count, "Result counts should match between QueryFilter and ExpressionFilter."); + CollectionAssert.AreEquivalent( + resultQueryFilter.Select(e => e.Age).ToList(), + resultExpressionFilter.Select(e => e.Age).ToList(), + "Result items should match between QueryFilter and ExpressionFilter." + ); + + // 3. Simulate combined filter: CurrentStatus == Inactive AND ManagerName == "Barbara" + var inactiveFilter = new List + { + new ScanCondition("CurrentStatus", ScanOperator.Equal, Status.Active), + new ScanCondition("ManagerName", ScanOperator.Equal, "Barbara") + }; + var contextExpressionBarbara = new ContextExpression(); + contextExpressionBarbara.SetFilter(e => e.ManagerName == "Barbara" && e.CurrentStatus == Status.Active); + + // Run each filter separately and take intersection + var resultActive = Context.Query("Diane", new QueryConfig + { + QueryFilter = inactiveFilter, + ConditionalOperator = ConditionalOperatorValues.And + }).ToList(); + var resultBarbara = Context.Query("Diane", new QueryConfig + { + ExpressionFilter = contextExpressionBarbara + }).ToList(); + + Assert.AreEqual(resultActive.Count, resultBarbara.Count, "Result counts should match between QueryFilter and ExpressionFilter."); + CollectionAssert.AreEquivalent( + resultActive.Select(e => e.Age).ToList(), + resultBarbara.Select(e => e.Age).ToList(), + "Result items should match between QueryFilter and ExpressionFilter." + ); + // 4. QueryFilter with ConditionalOperator.Or (CurrentStatus == Active OR Score == 101) + var orFilter = new List + { + new ScanCondition("CurrentStatus", ScanOperator.Equal, Status.Active), + new ScanCondition("Score", ScanOperator.Equal, 101) + }; + var resultOrQueryFilter = Context.Query("Diane", new QueryConfig + { + QueryFilter = orFilter, + ConditionalOperator = ConditionalOperatorValues.Or + }).ToList(); + + var contextExpressionOr = new ContextExpression(); + contextExpressionOr.SetFilter(e => e.CurrentStatus == Status.Active || e.Score == 101); + + var resultOrExpressionFilter = Context.Query("Diane", new QueryConfig + { + ExpressionFilter = contextExpressionOr + }).ToList(); + + // Assert both results are equivalent + Assert.AreEqual(resultOrQueryFilter.Count, resultOrExpressionFilter.Count, "Result counts should match between QueryFilter (OR) and ExpressionFilter (OR)."); + CollectionAssert.AreEquivalent( + resultOrQueryFilter.Select(e => e.Age).ToList(), + resultOrExpressionFilter.Select(e => e.Age).ToList(), + "Result items should match between QueryFilter (OR) and ExpressionFilter (OR)." + ); + + // 5. ExpressionFilter with index + var resultIndex = Context.Query("Big River", new QueryConfig + { + IndexName = "GlobalIndex", + ExpressionFilter = contextExpression + }).ToList(); + Assert.AreEqual(2, resultIndex.Count); + Assert.IsTrue(resultIndex.All(e => e.ManagerName == "Eva")); + } + /// /// Tests that the DynamoDB operations can read and write polymorphic items. /// diff --git a/sdk/test/Services/DynamoDBv2/UnitTests/Custom/DataModelOperationSpecificConfigTests.cs b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/DataModelOperationSpecificConfigTests.cs index cd568f7a1b61..42241a02a5f1 100644 --- a/sdk/test/Services/DynamoDBv2/UnitTests/Custom/DataModelOperationSpecificConfigTests.cs +++ b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/DataModelOperationSpecificConfigTests.cs @@ -261,7 +261,7 @@ public void QueryConfig() { // If this fails because you've added a property, be sure to add it to // `ToDynamoDBOperationConfig` before updating this unit test - Assert.AreEqual(10, typeof(QueryConfig).GetProperties().Length); + Assert.AreEqual(11, typeof(QueryConfig).GetProperties().Length); } [TestMethod] diff --git a/sdk/test/Services/DynamoDBv2/UnitTests/Custom/MockabilityTests/AsyncSearchTests.cs b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/MockabilityTests/AsyncSearchTests.cs index 825a203e9179..e2b8496429c8 100644 --- a/sdk/test/Services/DynamoDBv2/UnitTests/Custom/MockabilityTests/AsyncSearchTests.cs +++ b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/MockabilityTests/AsyncSearchTests.cs @@ -20,7 +20,7 @@ public async Task TestMockability_ScanAsync() .Returns(CreateMockAsyncSearch(new List { "item1", "item2" })); var ddbContext = mockContext.Object; - var asyncSearch = ddbContext.ScanAsync(null); + var asyncSearch = ddbContext.ScanAsync((IEnumerable)null); var results = await asyncSearch.GetNextSetAsync(); Assert.AreEqual(2, results.Count); From f2418a0b8a6eed42ee5d7c2440d7e4a11f3161b1 Mon Sep 17 00:00:00 2001 From: irina-herciu Date: Thu, 5 Jun 2025 16:50:19 +0300 Subject: [PATCH 02/13] wip --- .../Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs index dffca5ae491b..d7adeb39c6eb 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs @@ -1743,7 +1743,7 @@ private static ConstantExpression GetConstant(Expression expr) private static DynamoDBEntry ToAttributeValue(object value) { - //todo - add support for other types + //todo - remove this later return value switch { string s => s, @@ -2067,6 +2067,7 @@ private ContextSearch var valuesList = values?.ToList(); if (op == QueryOperator.Between && valuesList != null && valuesList.Count() == 2) { + //todo - use ToDynamoDBEntry to convert values to DynamoDBEntry keyExpression.ExpressionAttributeValues.Add(":rangeKey0", ToAttributeValue(valuesList.ElementAt(0))); keyExpression.ExpressionAttributeValues.Add(":rangeKey1", ToAttributeValue(valuesList.ElementAt(1))); } From 26753aed3a68de655004c8c4c0bd6c4c8fe88e3b Mon Sep 17 00:00:00 2001 From: irina-herciu Date: Fri, 6 Jun 2025 17:25:13 +0300 Subject: [PATCH 03/13] refactoring --- .../Custom/DataModel/ContextExpression.cs | 141 +- .../Custom/DataModel/ContextInternal.cs | 1395 ++++++++--------- .../Custom/DocumentModel/ExpressionBuilder.cs | 65 +- 3 files changed, 812 insertions(+), 789 deletions(-) diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs index b86b8073e6db..cd533f362ffc 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs @@ -1,6 +1,7 @@ -using System; +using Amazon.DynamoDBv2.DocumentModel; +using System; +using System.Collections.Generic; using System.Linq.Expressions; -using Amazon.DynamoDBv2.DocumentModel; using Expression = System.Linq.Expressions.Expression; namespace Amazon.DynamoDBv2.DataModel @@ -98,4 +99,140 @@ public PathNode(string path, int indexDepth, bool isMap, string formattedPath) FormattedPath = formattedPath; } } + + internal static class ContextExpressionsUtils + { + internal static string GetRangeKeyConditionExpression(string rangeKeyAlias, QueryOperator op) + { + return op switch + { + QueryOperator.Equal => $" AND {rangeKeyAlias} = :rangeKey0", + QueryOperator.LessThan => $" AND {rangeKeyAlias} < :rangeKey0", + QueryOperator.LessThanOrEqual => $" AND {rangeKeyAlias} <= :rangeKey0", + QueryOperator.GreaterThan => $" AND {rangeKeyAlias} > :rangeKey0", + QueryOperator.GreaterThanOrEqual => $" AND {rangeKeyAlias} >= :rangeKey0", + QueryOperator.Between => $" AND {rangeKeyAlias} BETWEEN :rangeKey0 AND :rangeKey0", + QueryOperator.BeginsWith => $" AND begins_with({rangeKeyAlias}, :rangeKey0)", + _ => throw new NotSupportedException($"QueryOperator '{op}' is not supported for key conditions.") + }; + } + + internal static bool IsMember(Expression expr) + { + return expr switch + { + MemberExpression memberExpr => true, + UnaryExpression ue => IsMember(ue.Operand), + _ => false + }; + } + + + internal static ConstantExpression GetConstant(Expression expr) + { + return expr switch + { + ConstantExpression constant => constant, + // If the expression is a UnaryExpression, check its Operand + UnaryExpression unary => unary.Operand as ConstantExpression, + NewExpression => throw new NotSupportedException($"Unsupported expression type {expr.Type}"), + _ => null + }; + } + + internal static bool IsComparison(ExpressionType type) + { + return type is ExpressionType.Equal or ExpressionType.NotEqual or + ExpressionType.GreaterThan or ExpressionType.GreaterThanOrEqual or + ExpressionType.LessThan or ExpressionType.LessThanOrEqual; + } + + internal static MemberExpression GetMember(Expression expr) + { + if (expr is MemberExpression memberExpr) + return memberExpr; + + if (expr is UnaryExpression ue) + return GetMember(ue.Operand); + + // Handle indexer access (get_Item) for lists/arrays/dictionaries + if (expr is MethodCallExpression methodCall && methodCall.Method.Name == "get_Item") + return GetMember(methodCall.Object); + + return null; + } + + internal static List ExtractPathNodes(Expression expr) + { + var pathNodes = new List(); + int indexDepth = 0; + string indexed = string.Empty; + + while (expr != null) + { + switch (expr) + { + case MemberExpression memberExpr: + pathNodes.Insert(0, + new PathNode(memberExpr.Member.Name, indexDepth, false, $"#n{indexed}")); + indexed = string.Empty; + indexDepth = 0; + expr = memberExpr.Expression; + break; + case MethodCallExpression { Method: { Name: "First" or "FirstOrDefault" } } methodCall: + expr = methodCall.Arguments.Count > 0 ? methodCall.Arguments[0] : methodCall.Object; + indexDepth++; + indexed += "[0]"; + break; + case MethodCallExpression { Method: { Name: "get_Item" } } methodCall: + { + var arg = methodCall.Arguments[0]; + if (arg is ConstantExpression constArg) + { + var indexValue = constArg.Value; + switch (indexValue) + { + case int intValue: + indexDepth++; + indexed += $"[{intValue}]"; + break; + case string stringValue: + pathNodes.Insert(0, new PathNode(stringValue, indexDepth, true, $"#n{indexed}")); + indexDepth = 0; + indexed = string.Empty; + break; + default: + throw new NotSupportedException( + $"Indexer argument must be an integer or string, got {indexValue.GetType().Name}."); + } + } + else + { + throw new NotSupportedException( + $"Method {methodCall.Method.Name} is not supported in property path."); + } + + expr = methodCall.Object; + break; + } + case MethodCallExpression methodCall: + throw new NotSupportedException( + $"Method {methodCall.Method.Name} is not supported in property path."); + case UnaryExpression + { + NodeType: ExpressionType.Convert or ExpressionType.ConvertChecked + } unaryExpr: + // Handle conversion expressions (e.g., (int)someEnum) + expr = unaryExpr.Operand; + break; + + default: + expr = null; + break; + } + } + + return pathNodes; + } + } } \ No newline at end of file diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs index d7adeb39c6eb..4f6cb7e5d9af 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Linq; @@ -1154,1017 +1154,876 @@ private static List CreateQueryConditions(DynamoDBFlatConfig fla return conditions; } - private DocumentModel.Expression ComposeExpression(Expression filterExpression, ItemStorageConfig storageConfig, - DynamoDBFlatConfig flatConfig) + // Key creation + private DynamoDBEntry ValueToDynamoDBEntry(PropertyStorage propertyStorage, object value, DynamoDBFlatConfig flatConfig) { - DocumentModel.Expression filter = new DocumentModel.Expression(); - if (filterExpression == null) return filter; - - - var aliasList = new KeyAttributeAliasList(); - var expressionNode = BuildExpressionNode(filterExpression, storageConfig, flatConfig); + var entry = ToDynamoDBEntry(propertyStorage, value, flatConfig); + return entry; + } + private static void ValidateKey(Key key, ItemStorageConfig storageConfig) + { + if (key == null) throw new ArgumentNullException("key"); + if (storageConfig == null) throw new ArgumentNullException("storageConfig"); + if (key.Count == 0) throw new InvalidOperationException("Key is empty"); - filter.ExpressionStatement = expressionNode.BuildExpressionString(aliasList, "C"); - if (aliasList.NamesList != null && aliasList.NamesList.Count != 0) + foreach (string hashKey in storageConfig.HashKeyPropertyNames) { - var namesDictionary = new Dictionary(); - for (int i = 0; i < aliasList.NamesList.Count; i++) - { - namesDictionary[$"#C{i}"] = aliasList.NamesList[i]; - } - - filter.ExpressionAttributeNames = namesDictionary; + string attributeName = storageConfig.BaseTypeStorageConfig.GetPropertyStorage(hashKey).AttributeName; + if (!key.ContainsKey(attributeName)) + throw new InvalidOperationException("Key missing hash key " + hashKey); } - - if (aliasList.ValuesList != null && aliasList.ValuesList.Count != 0) + foreach (string rangeKey in storageConfig.RangeKeyPropertyNames) { - var values = new Dictionary(); - for (int i = 0; i < aliasList.ValuesList.Count; i++) - { - values[$":C{i}"] = aliasList.ValuesList[i]; - } - - filter.ExpressionAttributeValues = values; + string attributeName = storageConfig.BaseTypeStorageConfig.GetPropertyStorage(rangeKey).AttributeName; + if (!key.ContainsKey(attributeName)) + throw new InvalidOperationException("Key missing range key " + rangeKey); } - - return filter; } - private ExpressionNode BuildExpressionNode(Expression expr, ItemStorageConfig storageConfig, - DynamoDBFlatConfig flatConfig) + internal Key MakeKey(object hashKey, object rangeKey, ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) { - var node = new ExpressionNode(); - - switch (expr) + if (storageConfig.HashKeyPropertyNames.Count != 1) { - case LambdaExpression lambda: - // Recursively process the body of the lambda - return BuildExpressionNode(lambda.Body, storageConfig, flatConfig); - case BinaryExpression binary when IsComparison(binary.NodeType): - node = HandleBinaryComparison(binary, storageConfig, flatConfig); - break; - - case BinaryExpression binary: - // Handle AND/OR expressions - var left = BuildExpressionNode(binary.Left, storageConfig, flatConfig); - var right = BuildExpressionNode(binary.Right, storageConfig, flatConfig); - node.Children.Enqueue(left); - node.Children.Enqueue(right); - var condition = binary.NodeType == ExpressionType.AndAlso ? "AND" : "OR"; - node.FormatedExpression = $"(#c) {condition} (#c)"; - break; - - case MethodCallExpression method: - node = HandleMethodCall(method, storageConfig, flatConfig); - break; - - case UnaryExpression { NodeType: ExpressionType.Not } unary: - var notUnary = BuildExpressionNode(unary.Operand, storageConfig, flatConfig); - node.Children.Enqueue(notUnary); - node.FormatedExpression = $"NOT (#c)"; - break; - - default: - throw new InvalidOperationException($"Unsupported expression type: {expr.NodeType}"); + var tableName = GetTableName(storageConfig.TableName, flatConfig); + throw new InvalidOperationException("Must have one hash key defined for the table " + tableName); } - return node; - } + Key key = new Key(); - private ExpressionNode HandleBinaryComparison(BinaryExpression expr, ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) - { - Expression member = null; - ConstantExpression constant = null; - - if (IsMember(expr.Left)) - { - member = expr.Left; - constant = GetConstant(expr.Right); - } - else if (IsMember(expr.Right)) - { - member = expr.Right; - constant = GetConstant(expr.Left); - } + string hashKeyPropertyName = storageConfig.HashKeyPropertyNames[0]; + PropertyStorage hashKeyProperty = storageConfig.BaseTypeStorageConfig.GetPropertyStorage(hashKeyPropertyName); - if (member == null) - throw new NotSupportedException("Expected member access"); + DynamoDBEntry hashKeyEntry = ValueToDynamoDBEntry(hashKeyProperty, hashKey, flatConfig); + if (hashKeyEntry == null) throw new InvalidOperationException("Unable to convert hash key value for property " + hashKeyPropertyName); + if (storageConfig.AttributesToStoreAsEpoch.Contains(hashKeyProperty.AttributeName)) + hashKeyEntry = Document.DateTimeToEpochSeconds(hashKeyEntry, hashKeyProperty.AttributeName); + if (storageConfig.AttributesToStoreAsEpochLong.Contains(hashKeyProperty.AttributeName)) + hashKeyEntry = Document.DateTimeToEpochSecondsLong(hashKeyEntry, hashKeyProperty.AttributeName); + var hashKeyEntryAttributeConversionConfig = new DynamoDBEntry.AttributeConversionConfig(flatConfig.Conversion, flatConfig.IsEmptyStringValueEnabled); + key[hashKeyProperty.AttributeName] = hashKeyEntry.ConvertToAttributeValue(hashKeyEntryAttributeConversionConfig); - var node = new ExpressionNode + if (storageConfig.RangeKeyPropertyNames.Count > 0) { - FormatedExpression = expr.NodeType switch + if (storageConfig.RangeKeyPropertyNames.Count != 1) { - ExpressionType.Equal => "#c = #c", - ExpressionType.NotEqual => "#c <> #c", - ExpressionType.LessThan => "#c < #c", - ExpressionType.LessThanOrEqual => "#c <= #c", - ExpressionType.GreaterThan => "#c > #c", - ExpressionType.GreaterThanOrEqual => "#c >= #c", - _ => throw new InvalidOperationException($"Unsupported mode: {expr.NodeType}") + var tableName = GetTableName(storageConfig.TableName, flatConfig); + throw new InvalidOperationException("Must have one range key defined for the table " + tableName); } - }; - SetExpressionNodeAttributes(storageConfig, member, constant, node, flatConfig); + string rangeKeyPropertyName = storageConfig.RangeKeyPropertyNames[0]; + PropertyStorage rangeKeyProperty = storageConfig.BaseTypeStorageConfig.GetPropertyStorage(rangeKeyPropertyName); - return node; - } + DynamoDBEntry rangeKeyEntry = ValueToDynamoDBEntry(rangeKeyProperty, rangeKey, flatConfig); + if (rangeKeyEntry == null) throw new InvalidOperationException("Unable to convert range key value for property " + rangeKeyPropertyName); + if (storageConfig.AttributesToStoreAsEpoch.Contains(rangeKeyProperty.AttributeName)) + rangeKeyEntry = Document.DateTimeToEpochSeconds(rangeKeyEntry, rangeKeyProperty.AttributeName); + if (storageConfig.AttributesToStoreAsEpochLong.Contains(rangeKeyProperty.AttributeName)) + rangeKeyEntry = Document.DateTimeToEpochSecondsLong(rangeKeyEntry, rangeKeyProperty.AttributeName); - private ExpressionNode HandleMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, - DynamoDBFlatConfig flatConfig) + var rangeKeyEntryAttributeConversionConfig = new DynamoDBEntry.AttributeConversionConfig(flatConfig.Conversion, flatConfig.IsEmptyStringValueEnabled); + key[rangeKeyProperty.AttributeName] = rangeKeyEntry.ConvertToAttributeValue(rangeKeyEntryAttributeConversionConfig); + } + + ValidateKey(key, storageConfig); + return key; + } + internal Key MakeKey(T keyObject, ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) { - // Handle method calls like Equals, Between, In, AttributeExists, AttributeNotExists, AttributeType, BeginsWith, Contains - return expr.Method.Name switch - { - "Equals" => HandleEqualsMethodCall(expr, storageConfig, flatConfig), - "Contains" => HandleContainsMethodCall(expr, storageConfig, flatConfig), - "StartsWith" => HandleStartsWithMethodCall(expr, storageConfig, flatConfig), - "In" => HandleInMethodCall(expr, storageConfig, flatConfig), - "Between" => HandleBetweenMethodCall(expr, storageConfig, flatConfig), - "AttributeExists" => HandleExistsMethodCall(expr, storageConfig, flatConfig), - "IsNull" or "AttributeNotExists" => HandleIsNullMethodCall(expr, storageConfig, flatConfig), - "AttributeType" => HandleAttributeTypeMethodCall(expr, storageConfig, flatConfig), - _ => throw new NotSupportedException($"Unsupported method call: {expr.Method.Name}") - }; + ItemStorage keyAsStorage = ObjectToItemStorageHelper(keyObject, storageConfig, flatConfig, keysOnly: true, ignoreNullValues: true); + if (storageConfig.HasVersion) // if version field is defined, it would have been returned, so remove before making the key + keyAsStorage.Document[storageConfig.VersionPropertyStorage.AttributeName] = null; + Key key = new Key(keyAsStorage.Document.ToAttributeMap(flatConfig.Conversion, storageConfig.AttributesToStoreAsEpoch, storageConfig.AttributesToStoreAsEpochLong, flatConfig.IsEmptyStringValueEnabled)); + ValidateKey(key, storageConfig); + return key; } - private ExpressionNode HandleAttributeTypeMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, - DynamoDBFlatConfig flatConfig) + // Searching + internal class ContextSearch { - var node = new ExpressionNode - { - FormatedExpression = "attribute_type (#c, #c)" - }; + public DynamoDBFlatConfig FlatConfig { get; set; } + public Search Search { get; set; } - if (expr.Arguments.Count == 2 && expr.Object == null) - { - if (expr.Arguments[0] is MemberExpression memberObj && - expr.Arguments[1] is ConstantExpression typeExpr) - { - SetExpressionNodeAttributes(storageConfig, memberObj, typeExpr, node, flatConfig); - } - else - { - throw new NotSupportedException("Expected MemberExpression and ConstantExpression as arguments for AttributeType method call."); - } - } - else + public ContextSearch(Search search, DynamoDBFlatConfig flatConfig) { - throw new NotSupportedException("Expected MemberExpression and ConstantExpression as arguments for AttributeType method call."); + Search = search; + FlatConfig = flatConfig; } - return node; } - private ExpressionNode HandleIsNullMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, - DynamoDBFlatConfig flatConfig) + private IEnumerable FromSearch<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ContextSearch cs) { - var node = new ExpressionNode { - FormatedExpression = "attribute_not_exists (#c)" - }; + if (cs == null) throw new ArgumentNullException("cs"); - if (expr.Arguments.Count == 1 && expr.Object == null) + // Configure search to not collect results + cs.Search.CollectResults = false; + + ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(cs.FlatConfig); + while (!cs.Search.IsDone) { - var collectionExpr = expr.Arguments[0] as MemberExpression; - if (collectionExpr != null) - { - SetExpressionNameNode(storageConfig, collectionExpr, node, flatConfig); - } - else + List set = cs.Search.GetNextSetHelper(); + foreach (var document in set) { - throw new NotSupportedException("Expected MemberExpression as argument for AttributeNotExists method call."); + ItemStorage storage = new ItemStorage(storageConfig); + storage.Document = document; + T instance = DocumentToObject(storage, cs.FlatConfig); + yield return instance; } } - else - { - throw new NotSupportedException("Expected MemberExpression as argument for AttributeNotExists method call."); - } - return node; + // Reset search to allow retrieving items more than once + cs.Search.Reset(); } - private ExpressionNode HandleExistsMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, - DynamoDBFlatConfig flatConfig) + #endregion + + #region Scan/Query + + private ContextSearch ConvertScan<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(IEnumerable conditions, DynamoDBOperationConfig operationConfig) { - var node = new ExpressionNode + DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, this.Config); + ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); + ScanFilter filter = ComposeScanFilter(conditions, storageConfig, flatConfig); + + Table table = GetTargetTable(storageConfig, flatConfig); + var scanConfig = new ScanOperationConfig { - FormatedExpression = "attribute_exists (#c)" + AttributesToGet = storageConfig.AttributesToGet, + Select = SelectValues.SpecificAttributes, + Filter = filter, + ConditionalOperator = flatConfig.ConditionalOperator, + IndexName = flatConfig.IndexName, + ConsistentRead = flatConfig.ConsistentRead.GetValueOrDefault(false) }; - if (expr.Arguments.Count == 1 && expr.Object == null) + // table.Scan() returns the ISearch interface but we explicitly cast it to a Search object since we rely on its internal behavior + Search scan = table.Scan(scanConfig) as Search; + return new ContextSearch(scan, flatConfig); + } + + + private ContextSearch ConvertScan(ContextExpression filterExpression, DynamoDBOperationConfig operationConfig) + { + DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, this.Config); + ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); + + DocumentModel.Expression expression = null; + if (filterExpression is { Filter: null }) { - var collectionExpr = expr.Arguments[0] as MemberExpression; - if (collectionExpr != null) - { - SetExpressionNameNode(storageConfig, collectionExpr, node, flatConfig); - } - else - { - throw new NotSupportedException("Expected MemberExpression as argument for AttributeExists method call."); - } + expression = ComposeExpression(filterExpression.Filter, storageConfig, flatConfig); } - return node; + Table table = GetTargetTable(storageConfig, flatConfig); + var scanConfig = new ScanOperationConfig + { + AttributesToGet = storageConfig.AttributesToGet, + Select = SelectValues.SpecificAttributes, + FilterExpression = expression, + IndexName = flatConfig.IndexName, + ConsistentRead = flatConfig.ConsistentRead.GetValueOrDefault(false) + }; + + // table.Scan() returns the ISearch interface but we explicitly cast it to a Search object since we rely on its internal behavior + Search scan = table.Scan(scanConfig) as Search; + return new ContextSearch(scan, flatConfig); } - private ExpressionNode HandleInMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, - DynamoDBFlatConfig flatConfig) + private ContextSearch ConvertFromScan<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ScanOperationConfig scanConfig, DynamoDBOperationConfig operationConfig) { - var node = new ExpressionNode - { - FormatedExpression = "#c IN (" - }; + DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); + ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); + Table table = GetTargetTable(storageConfig, flatConfig); - if (expr.Object is MemberExpression memberObj && expr.Arguments[0] is NewArrayExpression arrayExpr) - { - var propertyStorage = SetExpressionNameNode(storageConfig, memberObj, node, flatConfig); + // table.Scan() returns the ISearch interface but we explicitly cast it to a Search object since we rely on its internal behavior + Search search = table.Scan(scanConfig) as Search; + return new ContextSearch(search, flatConfig); + } - foreach (var arg in arrayExpr.Expressions) - { - if (arg is not ConstantExpression constExpr) continue; + private ContextSearch ConvertFromQuery<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(QueryOperationConfig queryConfig, DynamoDBOperationConfig operationConfig) + { + DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); + ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); + Table table = GetTargetTable(storageConfig, flatConfig); - node.FormatedExpression += "#c, "; + // table.Query() returns the ISearch interface but we explicitly cast it to a Search object since we rely on its internal behavior + Search search = table.Query(queryConfig) as Search; + return new ContextSearch(search, flatConfig); + } - SetExpressionValueNode(constExpr, node, propertyStorage, flatConfig); - } - } - else + private ContextSearch ConvertQueryByValue<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(object hashKeyValue, QueryOperator op, IEnumerable values, DynamoDBOperationConfig operationConfig) + { + if (operationConfig!=null) { - throw new NotSupportedException("Expected MemberExpression with NewArrayExpression as argument for In method call."); + operationConfig.ValidateFilter(); } - if (node.FormatedExpression.EndsWith(", ")) + DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); + ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); + //todo - add support for expression + ContextSearch query; + if (operationConfig is { ExpressionFilter: { Filter: not null } }) { - node.FormatedExpression = node.FormatedExpression.Substring(0, node.FormatedExpression.Length - 2); + query = ConvertQueryByValueWithExpression(hashKeyValue, op, values, operationConfig.ExpressionFilter.Filter, operationConfig, storageConfig); } - node.FormatedExpression += ")"; - return node; + else + { + List conditions = CreateQueryConditions(flatConfig, op, values, storageConfig); + query = ConvertQueryByValue(hashKeyValue, conditions, operationConfig, storageConfig); + } + return query; } - private ExpressionNode HandleBetweenMethodCall(MethodCallExpression expr, - ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) + private ContextSearch ConvertQueryByValueWithExpression(object hashKeyValue, QueryOperator op, IEnumerable values, + Expression filterExpression, DynamoDBOperationConfig operationConfig, ItemStorageConfig storageConfig) { - var node = new ExpressionNode - { - FormatedExpression = "#c BETWEEN #c AND #c" - }; - - - if (expr.Arguments.Count == 3 && expr.Object == null) - { - var collectionExpr = expr.Arguments[0] as MemberExpression; - var constExprLeft = expr.Arguments[1] as ConstantExpression; - var constExprRight = expr.Arguments[2] as ConstantExpression; + DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); - if (collectionExpr != null && constExprLeft != null && constExprRight != null) - { - var propertyStorage = SetExpressionNameNode(storageConfig, collectionExpr, node, flatConfig); - SetExpressionValueNode(constExprLeft, node, propertyStorage, flatConfig); - SetExpressionValueNode(constExprRight, node, propertyStorage, flatConfig); - } - } - else + if (storageConfig == null) + storageConfig = StorageConfigCache.GetConfig(flatConfig); + if (operationConfig.QueryFilter != null && operationConfig.QueryFilter.Count != 0) { - throw new NotSupportedException("Expected MemberExpression with NewArrayExpression as argument for In method call."); + throw new InvalidOperationException("QueryFilter is not supported with filter expression. Use either QueryFilter or filter expression, but not both."); } + return ConvertQueryHelper(hashKeyValue, op, values, flatConfig, storageConfig, filterExpression); - return node; } - private ExpressionNode HandleStartsWithMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, - DynamoDBFlatConfig flatConfig) + private ContextSearch + ConvertQueryByValue<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>( + object hashKeyValue, IEnumerable conditions, DynamoDBOperationConfig operationConfig, + ItemStorageConfig storageConfig = null) { - var node = new ExpressionNode + // DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); + // if (storageConfig == null) + // storageConfig = StorageConfigCache.GetConfig(flatConfig); + + // List indexNames; + // QueryFilter filter = ComposeQueryFilter(flatConfig, hashKeyValue, conditions, storageConfig, out indexNames); + // return ConvertQueryHelper(flatConfig, storageConfig, filter, indexNames); + + if (operationConfig != null) { - FormatedExpression = "begins_with (#c, #c)" - }; - if (expr.Object is MemberExpression memberObj && expr.Arguments[0] is ConstantExpression argConst) + operationConfig.ValidateFilter(); + } + + DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); + + if (storageConfig == null) + storageConfig = StorageConfigCache.GetConfig(flatConfig); + + ContextSearch query; + if (operationConfig is { ExpressionFilter: { Filter: not null } }) { - SetExpressionNodeAttributes(storageConfig, memberObj, argConst, node,flatConfig); + query = ConvertQueryByValueWithExpression(hashKeyValue, QueryOperator.Equal, null, + operationConfig.ExpressionFilter.Filter, operationConfig, storageConfig); } else { - throw new NotSupportedException("Expected MemberExpression with ConstantExpression as argument for StartsWith method call."); + + List indexNames; + QueryFilter filter = ComposeQueryFilter(flatConfig, hashKeyValue, conditions, storageConfig, out indexNames); + query = ConvertQueryHelper(flatConfig, storageConfig, filter, indexNames); } - return node; + return query; } - private ExpressionNode HandleContainsMethodCall(MethodCallExpression expr, - ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) + private ContextSearch ConvertQueryHelper<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>( + object hashKeyValue, QueryOperator op, IEnumerable values, DynamoDBFlatConfig flatConfig, + ItemStorageConfig storageConfig, + Expression filterExpression) { - var node = new ExpressionNode - { - FormatedExpression = "contains (#c, #c)" - }; - if (expr.Object is MemberExpression memberObj && expr.Arguments[0] is ConstantExpression argConst) - { - SetExpressionNodeAttributes(storageConfig, memberObj, argConst, node,flatConfig); - } - else if (expr.Arguments.Count == 2 && expr.Object == null) + ValidateHashKey(hashKeyValue, storageConfig); + ValidateQueryKeyConfiguration(storageConfig, flatConfig); + + var hashKeyEntry = HashKeyValueToDynamoDBEntry(flatConfig, hashKeyValue, storageConfig); + var keyExpression = new DocumentModel.Expression { - var collectionExpr = expr.Arguments[0] as MemberExpression; - var constExpr = expr.Arguments[1] as ConstantExpression; + ExpressionStatement = "#hashKey = :hashKey", + ExpressionAttributeValues = new Dictionary + { + { ":hashKey", hashKeyEntry.Item2 } + }, + ExpressionAttributeNames = new Dictionary + { + { "#hashKey", hashKeyEntry.Item1 } + } + }; - if (collectionExpr != null && constExpr != null) + string rangeKeyPropertyName; + + string indexName = flatConfig.IndexName; + if (string.IsNullOrEmpty(indexName)) + rangeKeyPropertyName = storageConfig.RangeKeyPropertyNames.FirstOrDefault(); + else + rangeKeyPropertyName = storageConfig.GetRangeKeyByIndex(indexName); + + if (!string.IsNullOrEmpty(rangeKeyPropertyName) && values!=null) + { + //todo implement QueryOperator to expression mapping + keyExpression.ExpressionStatement += ContextExpressionsUtils.GetRangeKeyConditionExpression($"#rangeKey", op); + keyExpression.ExpressionAttributeNames.Add("#rangeKey", rangeKeyPropertyName); + var valuesList = values?.ToList(); + var rangeKeyProperty = storageConfig.BaseTypeStorageConfig.GetPropertyStorage(rangeKeyPropertyName); + if (op == QueryOperator.Between && valuesList != null && valuesList.Count() == 2) { - SetExpressionNodeAttributes(storageConfig, collectionExpr, constExpr, node,flatConfig); + keyExpression.ExpressionAttributeValues.Add(":rangeKey0", ToDynamoDBEntry( + rangeKeyProperty, + valuesList.ElementAt(0), + flatConfig, + true)); + keyExpression.ExpressionAttributeValues.Add(":rangeKey1", ToDynamoDBEntry( + rangeKeyProperty, + valuesList.ElementAt(1), + flatConfig, + true)); } else { - throw new NotSupportedException( - "Expected MemberExpression with ConstantExpression as argument for Contains method call."); + keyExpression.ExpressionAttributeValues.Add(":rangeKey0", ToDynamoDBEntry( + rangeKeyProperty, + valuesList.FirstOrDefault(), + flatConfig, + true + )); } } + + Table table = GetTargetTable(storageConfig, flatConfig); + var queryConfig = new QueryOperationConfig + { + ConsistentRead = flatConfig.ConsistentRead.Value, + BackwardSearch = flatConfig.BackwardQuery.Value, + KeyExpression = keyExpression, + }; + + var expression = ComposeExpression(filterExpression, storageConfig, flatConfig); + + //TODO string indexName = GetQueryIndexName(currentConfig, indexNames); + queryConfig.FilterExpression = expression; + + if (string.IsNullOrEmpty(indexName)) + { + queryConfig.Select = SelectValues.SpecificAttributes; + List attributesToGet = storageConfig.AttributesToGet; + queryConfig.AttributesToGet = attributesToGet; + } else { - throw new NotSupportedException( - "Expected MemberExpression with ConstantExpression as argument for Contains method call."); + queryConfig.IndexName = indexName; + queryConfig.Select = SelectValues.AllProjectedAttributes; } + Search query = table.Query(queryConfig) as Search; - return node; + return new ContextSearch(query, flatConfig); } - private ExpressionNode HandleEqualsMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, - DynamoDBFlatConfig flatConfig) + + + private ContextSearch ConvertQueryHelper(DynamoDBFlatConfig currentConfig, ItemStorageConfig storageConfig, QueryFilter filter, List indexNames) { - var node = new ExpressionNode + Table table = GetTargetTable(storageConfig, currentConfig); + string indexName = GetQueryIndexName(currentConfig, indexNames); + var queryConfig = new QueryOperationConfig { - FormatedExpression = $"#c = #c" + Filter = filter, + ConsistentRead = currentConfig.ConsistentRead.Value, + BackwardSearch = currentConfig.BackwardQuery.Value, + IndexName = indexName, + ConditionalOperator = currentConfig.ConditionalOperator }; - - if (expr.Object is MemberExpression member && - expr.Arguments[0] is ConstantExpression constant && - constant.Value == null) + if (string.IsNullOrEmpty(indexName)) { - SetExpressionNodeAttributes(storageConfig, member, constant, node, flatConfig); - return node; + queryConfig.Select = SelectValues.SpecificAttributes; + List attributesToGet = storageConfig.AttributesToGet; + queryConfig.AttributesToGet = attributesToGet; } - else if (expr.Arguments.Count == 2 && expr.Object == null) + else { - var memberObj = GetMember(expr.Arguments[0]) ?? GetMember(expr.Arguments[1]); - var argConst = GetConstant(expr.Arguments[1]) ?? GetConstant(expr.Arguments[0]); - if (memberObj != null && argConst != null) - { - SetExpressionNodeAttributes(storageConfig, memberObj, argConst, node, flatConfig); - return node; - } + queryConfig.Select = SelectValues.AllProjectedAttributes; } - throw new NotSupportedException("Expected MemberExpression with ConstantExpression as argument for Equals method call."); - } + // table.Query() returns the ISearch interface but we explicitly cast it to a Search object since we rely on its internal behavior + Search query = table.Query(queryConfig) as Search; - private void SetExpressionNodeAttributes(ItemStorageConfig storageConfig, Expression memberObj, - ConstantExpression argConst, ExpressionNode node, DynamoDBFlatConfig flatConfig) - { - var propertyStorage = SetExpressionNameNode(storageConfig, memberObj, node, flatConfig); - SetExpressionValueNode(argConst, node, propertyStorage, flatConfig); + return new ContextSearch(query, currentConfig); } - private void SetExpressionValueNode(ConstantExpression argConst, ExpressionNode node, PropertyStorage propertyStorage, DynamoDBFlatConfig flatConfig) + private AsyncSearch FromSearchAsync<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ContextSearch contextSearch) { - DynamoDBEntry entry=ToDynamoDBEntry(propertyStorage, argConst?.Value, flatConfig, canReturnScalarInsteadOfList: true); - var valuesNode = new ExpressionNode() - { - FormatedExpression = $"#v" - }; - valuesNode.Values.Enqueue(entry); - node.Children.Enqueue(valuesNode); + return new AsyncSearch(this, contextSearch); } - private PropertyStorage ResolveNestedPropertyStorage(StorageConfig rootConfig, DynamoDBFlatConfig flatConfig, - List path, Queue namesNodeNames) - { - StorageConfig currentConfig = rootConfig; - PropertyStorage propertyStorage= null; - for (int i = 0; i < path.Count; i++) - { - var pathNode = path[i]; - - // If the path node is a map, just add the name to the queue - if (pathNode.IsMap) - { - namesNodeNames.Enqueue(pathNode.Path); - continue; - } - - propertyStorage = currentConfig.GetPropertyStorage(pathNode.Path); - if (propertyStorage == null) - throw new InvalidOperationException($"Property '{pathNode.Path}' not found in storage config."); - // If the property is ignored, throw an exception - if (propertyStorage.IsIgnored) - { - throw new InvalidOperationException($"Property '{pathNode.Path}' is marked as ignored and cannot be used in a filter expression."); - } - - namesNodeNames.Enqueue(propertyStorage.AttributeName); - // If not the last segment, descend into the nested StorageConfig - if (i >= path.Count - 1) continue; + #endregion - // Only descend if the property is a complex type (not primitive/string) - var propertyType = propertyStorage.MemberType; - if (Utils.IsPrimitive(propertyType)) - throw new InvalidOperationException($"Property '{pathNode.Path}' is not a complex type."); + #region Expression Building - // Determine the element type if the property is a collection - var nextPathNode = path[i + 1]; + private DocumentModel.Expression ComposeExpression(Expression filterExpression, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) + { + DocumentModel.Expression filter = new DocumentModel.Expression(); + if (filterExpression == null) return filter; - Type elementType = null; - var depth = pathNode.IndexDepth; - if (nextPathNode is { IsMap: true }) - { - depth += nextPathNode.IndexDepth; - } - var nodePropertyType = propertyType; - var currentDepth = 0; + var aliasList = new KeyAttributeAliasList(); + var expressionNode = BuildExpressionNode(filterExpression, storageConfig, flatConfig); - while (currentDepth <= depth && nodePropertyType != null && Utils.ImplementsInterface(nodePropertyType, typeof(ICollection<>)) - && nodePropertyType != typeof(string)) + filter.ExpressionStatement = expressionNode.BuildExpressionString(aliasList, "C"); + if (aliasList.NamesList != null && aliasList.NamesList.Count != 0) + { + var namesDictionary = new Dictionary(); + for (int i = 0; i < aliasList.NamesList.Count; i++) { - elementType = Utils.GetElementType(nodePropertyType); - if (elementType == null) - { - IsSupportedDictionaryType(nodePropertyType, out elementType); - } - nodePropertyType = elementType; - currentDepth++; + namesDictionary[$"#C{i}"] = aliasList.NamesList[i]; } - elementType ??= propertyType; - - ItemStorageConfig config = StorageConfigCache.GetConfig(elementType, flatConfig); - currentConfig = config.BaseTypeStorageConfig; - } - - return propertyStorage; - } - private PropertyStorage SetExpressionNameNode(ItemStorageConfig storageConfig, Expression memberObj, - ExpressionNode node, DynamoDBFlatConfig flatConfig) - { - var path = ExtractPathNodes(memberObj); - if(path.Count == 0) - { - throw new InvalidOperationException("Expected a valid property path in the expression."); + filter.ExpressionAttributeNames = namesDictionary; } - var namesNode = new ExpressionNode() - { - FormatedExpression = string.Join(".", path.Select(pn => pn.FormattedPath)) - }; - var propertyStorage = ResolveNestedPropertyStorage(storageConfig.BaseTypeStorageConfig, flatConfig, path, namesNode.Names); - node.Children.Enqueue(namesNode); - - return propertyStorage; - - List ExtractPathNodes(Expression expr) + if (aliasList.ValuesList != null && aliasList.ValuesList.Count != 0) { - var pathNodes = new List(); - int indexDepth = 0; - string indexed = string.Empty; - - while (expr != null) + var values = new Dictionary(); + for (int i = 0; i < aliasList.ValuesList.Count; i++) { - switch (expr) - { - case MemberExpression memberExpr: - pathNodes.Insert(0, new PathNode(memberExpr.Member.Name, indexDepth, false, $"#n{indexed}")); - indexed = string.Empty; - indexDepth = 0; - expr = memberExpr.Expression; - break; - case MethodCallExpression methodCall - when methodCall.Method.Name == "First" || methodCall.Method.Name == "FirstOrDefault": - expr = methodCall.Arguments.Count > 0 ? methodCall.Arguments[0] : methodCall.Object; - indexDepth++; - indexed += "[0]"; - break; - case MethodCallExpression methodCall - when methodCall.Method.Name == "get_Item": - { - var arg = methodCall.Arguments[0]; - if (arg is ConstantExpression constArg) - { - var indexValue = constArg.Value; - switch (indexValue) - { - case int intValue: - indexDepth++; - indexed += $"[{intValue}]"; - break; - case string stringValue: - pathNodes.Insert(0, new PathNode(stringValue, indexDepth, true, $"#n{indexed}")); - indexDepth = 0; - indexed = string.Empty; - break; - default: - throw new NotSupportedException( - $"Indexer argument must be an integer or string, got {indexValue.GetType().Name}."); - } - } - else - { - throw new NotSupportedException( - $"Method {methodCall.Method.Name} is not supported in property path."); - } - - expr = methodCall.Object; - break; - } - case MethodCallExpression methodCall: - throw new NotSupportedException( - $"Method {methodCall.Method.Name} is not supported in property path."); - case UnaryExpression unaryExpr - when unaryExpr.NodeType == ExpressionType.Convert || unaryExpr.NodeType == ExpressionType.ConvertChecked: - // Handle conversion expressions (e.g., (int)someEnum) - expr = unaryExpr.Operand; - break; - - default: - expr = null; - break; - } + values[$":C{i}"] = aliasList.ValuesList[i]; } - return pathNodes; + filter.ExpressionAttributeValues = values; } - } - private static bool IsComparison(ExpressionType type) - { - return type is ExpressionType.Equal or ExpressionType.NotEqual or - ExpressionType.GreaterThan or ExpressionType.GreaterThanOrEqual or - ExpressionType.LessThan or ExpressionType.LessThanOrEqual; + return filter; } - private static MemberExpression GetMember(Expression expr) + private ExpressionNode BuildExpressionNode(Expression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) { - if (expr is MemberExpression memberExpr) - return memberExpr; + var node = new ExpressionNode(); - if (expr is UnaryExpression ue) - return GetMember(ue.Operand); + switch (expr) + { + case LambdaExpression lambda: + // Recursively process the body of the lambda + return BuildExpressionNode(lambda.Body, storageConfig, flatConfig); + case BinaryExpression binary when ContextExpressionsUtils.IsComparison(binary.NodeType): + node = HandleBinaryComparison(binary, storageConfig, flatConfig); + break; - // Handle indexer access (get_Item) for lists/arrays/dictionaries - if (expr is MethodCallExpression methodCall && methodCall.Method.Name == "get_Item") - return GetMember(methodCall.Object); + case BinaryExpression binary: + // Handle AND/OR expressions + var left = BuildExpressionNode(binary.Left, storageConfig, flatConfig); + var right = BuildExpressionNode(binary.Right, storageConfig, flatConfig); + node.Children.Enqueue(left); + node.Children.Enqueue(right); + var condition = binary.NodeType == ExpressionType.AndAlso ? "AND" : "OR"; + node.FormatedExpression = $"(#c) {condition} (#c)"; + break; - return null; - } + case MethodCallExpression method: + node = HandleMethodCall(method, storageConfig, flatConfig); + break; - private static bool IsMember(Expression expr) - { - if (expr is MemberExpression memberExpr) - return true; + case UnaryExpression { NodeType: ExpressionType.Not } unary: + var notUnary = BuildExpressionNode(unary.Operand, storageConfig, flatConfig); + node.Children.Enqueue(notUnary); + node.FormatedExpression = ExpressionFormatConstants.Not; + break; - if (expr is UnaryExpression ue) - return IsMember(ue.Operand); + default: + throw new InvalidOperationException($"Unsupported expression type: {expr.NodeType}"); + } - return false; + return node; } - - private static ConstantExpression GetConstant(Expression expr) + private ExpressionNode HandleBinaryComparison(BinaryExpression expr, ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) { - var constant = expr as ConstantExpression; - if (constant != null) - return constant; - // If the expression is a UnaryExpression, check its Operand - var unary = expr as UnaryExpression; - if (unary != null) + Expression member = null; + ConstantExpression constant = null; + + if (ContextExpressionsUtils.IsMember(expr.Left)) { - return unary.Operand as ConstantExpression; + member = expr.Left; + constant = ContextExpressionsUtils.GetConstant(expr.Right); } - var newexp= expr as NewExpression; - if (newexp != null) + else if (ContextExpressionsUtils.IsMember(expr.Right)) { - throw new NotSupportedException($"Unsupported expression type {expr.Type}"); + member = expr.Right; + constant = ContextExpressionsUtils.GetConstant(expr.Left); } - return null; + + if (member == null) + throw new NotSupportedException("Expected member access"); + + var node = new ExpressionNode + { + FormatedExpression = expr.NodeType switch + { + ExpressionType.Equal => ExpressionFormatConstants.Equal, + ExpressionType.NotEqual => ExpressionFormatConstants.NotEqual, + ExpressionType.LessThan => ExpressionFormatConstants.LessThan, + ExpressionType.LessThanOrEqual => ExpressionFormatConstants.LessThanOrEqual, + ExpressionType.GreaterThan => ExpressionFormatConstants.GreaterThan, + ExpressionType.GreaterThanOrEqual => ExpressionFormatConstants.GreaterThanOrEqual, + _ => throw new InvalidOperationException($"Unsupported mode: {expr.NodeType}") + } + }; + + SetExpressionNodeAttributes(storageConfig, member, constant, node, flatConfig); + + return node; } - private static DynamoDBEntry ToAttributeValue(object value) + private ExpressionNode HandleMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) { - //todo - remove this later - return value switch + // Handle method calls like Equals, Between, In, AttributeExists, AttributeNotExists, AttributeType, BeginsWith, Contains + return expr.Method.Name switch { - string s => s, - int i => i, - long l => l, - double d => d, - bool b => b, - _ => throw new NotSupportedException($"Unsupported value type: {value.GetType().Name}") + "Equals" => HandleEqualsMethodCall(expr, storageConfig, flatConfig), + "Contains" => HandleContainsMethodCall(expr, storageConfig, flatConfig), + "StartsWith" => HandleStartsWithMethodCall(expr, storageConfig, flatConfig), + "In" => HandleInMethodCall(expr, storageConfig, flatConfig), + "Between" => HandleBetweenMethodCall(expr, storageConfig, flatConfig), + "AttributeExists" => HandleExistsMethodCall(expr, storageConfig, flatConfig), + "IsNull" or "AttributeNotExists" => HandleIsNullMethodCall(expr, storageConfig, flatConfig), + "AttributeType" => HandleAttributeTypeMethodCall(expr, storageConfig, flatConfig), + _ => throw new NotSupportedException($"Unsupported method call: {expr.Method.Name}") }; } - // Key creation - private DynamoDBEntry ValueToDynamoDBEntry(PropertyStorage propertyStorage, object value, DynamoDBFlatConfig flatConfig) - { - var entry = ToDynamoDBEntry(propertyStorage, value, flatConfig); - return entry; - } - private static void ValidateKey(Key key, ItemStorageConfig storageConfig) + private ExpressionNode HandleAttributeTypeMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) { - if (key == null) throw new ArgumentNullException("key"); - if (storageConfig == null) throw new ArgumentNullException("storageConfig"); - if (key.Count == 0) throw new InvalidOperationException("Key is empty"); + var node = new ExpressionNode + { + FormatedExpression = ExpressionFormatConstants.AttributeType + }; - foreach (string hashKey in storageConfig.HashKeyPropertyNames) + if (expr.Arguments.Count == 2 && expr.Object == null) { - string attributeName = storageConfig.BaseTypeStorageConfig.GetPropertyStorage(hashKey).AttributeName; - if (!key.ContainsKey(attributeName)) - throw new InvalidOperationException("Key missing hash key " + hashKey); + if (expr.Arguments[0] is MemberExpression memberObj && + expr.Arguments[1] is ConstantExpression typeExpr) + { + SetExpressionNodeAttributes(storageConfig, memberObj, typeExpr, node, flatConfig); + } + else + { + throw new NotSupportedException("Expected MemberExpression and ConstantExpression as arguments for AttributeType method call."); + } } - foreach (string rangeKey in storageConfig.RangeKeyPropertyNames) + else { - string attributeName = storageConfig.BaseTypeStorageConfig.GetPropertyStorage(rangeKey).AttributeName; - if (!key.ContainsKey(attributeName)) - throw new InvalidOperationException("Key missing range key " + rangeKey); + throw new NotSupportedException("Expected MemberExpression and ConstantExpression as arguments for AttributeType method call."); } + return node; } - internal Key MakeKey(object hashKey, object rangeKey, ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) + private ExpressionNode HandleIsNullMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) { - if (storageConfig.HashKeyPropertyNames.Count != 1) + var node = new ExpressionNode { - var tableName = GetTableName(storageConfig.TableName, flatConfig); - throw new InvalidOperationException("Must have one hash key defined for the table " + tableName); - } - - Key key = new Key(); - - string hashKeyPropertyName = storageConfig.HashKeyPropertyNames[0]; - PropertyStorage hashKeyProperty = storageConfig.BaseTypeStorageConfig.GetPropertyStorage(hashKeyPropertyName); - - DynamoDBEntry hashKeyEntry = ValueToDynamoDBEntry(hashKeyProperty, hashKey, flatConfig); - if (hashKeyEntry == null) throw new InvalidOperationException("Unable to convert hash key value for property " + hashKeyPropertyName); - if (storageConfig.AttributesToStoreAsEpoch.Contains(hashKeyProperty.AttributeName)) - hashKeyEntry = Document.DateTimeToEpochSeconds(hashKeyEntry, hashKeyProperty.AttributeName); - if (storageConfig.AttributesToStoreAsEpochLong.Contains(hashKeyProperty.AttributeName)) - hashKeyEntry = Document.DateTimeToEpochSecondsLong(hashKeyEntry, hashKeyProperty.AttributeName); - var hashKeyEntryAttributeConversionConfig = new DynamoDBEntry.AttributeConversionConfig(flatConfig.Conversion, flatConfig.IsEmptyStringValueEnabled); - key[hashKeyProperty.AttributeName] = hashKeyEntry.ConvertToAttributeValue(hashKeyEntryAttributeConversionConfig); + FormatedExpression = ExpressionFormatConstants.AttributeNotExists + }; - if (storageConfig.RangeKeyPropertyNames.Count > 0) + if (expr.Arguments.Count == 1 && expr.Object == null) { - if (storageConfig.RangeKeyPropertyNames.Count != 1) + var collectionExpr = expr.Arguments[0] as MemberExpression; + if (collectionExpr != null) { - var tableName = GetTableName(storageConfig.TableName, flatConfig); - throw new InvalidOperationException("Must have one range key defined for the table " + tableName); + SetExpressionNameNode(storageConfig, collectionExpr, node, flatConfig); + } + else + { + throw new NotSupportedException("Expected MemberExpression as argument for AttributeNotExists method call."); } - - string rangeKeyPropertyName = storageConfig.RangeKeyPropertyNames[0]; - PropertyStorage rangeKeyProperty = storageConfig.BaseTypeStorageConfig.GetPropertyStorage(rangeKeyPropertyName); - - DynamoDBEntry rangeKeyEntry = ValueToDynamoDBEntry(rangeKeyProperty, rangeKey, flatConfig); - if (rangeKeyEntry == null) throw new InvalidOperationException("Unable to convert range key value for property " + rangeKeyPropertyName); - if (storageConfig.AttributesToStoreAsEpoch.Contains(rangeKeyProperty.AttributeName)) - rangeKeyEntry = Document.DateTimeToEpochSeconds(rangeKeyEntry, rangeKeyProperty.AttributeName); - if (storageConfig.AttributesToStoreAsEpochLong.Contains(rangeKeyProperty.AttributeName)) - rangeKeyEntry = Document.DateTimeToEpochSecondsLong(rangeKeyEntry, rangeKeyProperty.AttributeName); - - var rangeKeyEntryAttributeConversionConfig = new DynamoDBEntry.AttributeConversionConfig(flatConfig.Conversion, flatConfig.IsEmptyStringValueEnabled); - key[rangeKeyProperty.AttributeName] = rangeKeyEntry.ConvertToAttributeValue(rangeKeyEntryAttributeConversionConfig); } - - ValidateKey(key, storageConfig); - return key; - } - internal Key MakeKey(T keyObject, ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) - { - ItemStorage keyAsStorage = ObjectToItemStorageHelper(keyObject, storageConfig, flatConfig, keysOnly: true, ignoreNullValues: true); - if (storageConfig.HasVersion) // if version field is defined, it would have been returned, so remove before making the key - keyAsStorage.Document[storageConfig.VersionPropertyStorage.AttributeName] = null; - Key key = new Key(keyAsStorage.Document.ToAttributeMap(flatConfig.Conversion, storageConfig.AttributesToStoreAsEpoch, storageConfig.AttributesToStoreAsEpochLong, flatConfig.IsEmptyStringValueEnabled)); - ValidateKey(key, storageConfig); - return key; - } - - // Searching - internal class ContextSearch - { - public DynamoDBFlatConfig FlatConfig { get; set; } - public Search Search { get; set; } - - public ContextSearch(Search search, DynamoDBFlatConfig flatConfig) + else { - Search = search; - FlatConfig = flatConfig; + throw new NotSupportedException("Expected MemberExpression as argument for AttributeNotExists method call."); } + + return node; } - private IEnumerable FromSearch<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ContextSearch cs) + private ExpressionNode HandleExistsMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) { - if (cs == null) throw new ArgumentNullException("cs"); - - // Configure search to not collect results - cs.Search.CollectResults = false; + var node = new ExpressionNode + { + FormatedExpression = ExpressionFormatConstants.AttributeExists + }; - ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(cs.FlatConfig); - while (!cs.Search.IsDone) + if (expr.Arguments.Count == 1 && expr.Object == null) { - List set = cs.Search.GetNextSetHelper(); - foreach (var document in set) + var collectionExpr = expr.Arguments[0] as MemberExpression; + if (collectionExpr != null) { - ItemStorage storage = new ItemStorage(storageConfig); - storage.Document = document; - T instance = DocumentToObject(storage, cs.FlatConfig); - yield return instance; + SetExpressionNameNode(storageConfig, collectionExpr, node, flatConfig); + } + else + { + throw new NotSupportedException("Expected MemberExpression as argument for AttributeExists method call."); } } - // Reset search to allow retrieving items more than once - cs.Search.Reset(); + return node; } - #endregion - - #region Scan/Query - - private ContextSearch ConvertScan<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(IEnumerable conditions, DynamoDBOperationConfig operationConfig) + private ExpressionNode HandleInMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) { - DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, this.Config); - ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); - ScanFilter filter = ComposeScanFilter(conditions, storageConfig, flatConfig); - - Table table = GetTargetTable(storageConfig, flatConfig); - var scanConfig = new ScanOperationConfig + var node = new ExpressionNode { - AttributesToGet = storageConfig.AttributesToGet, - Select = SelectValues.SpecificAttributes, - Filter = filter, - ConditionalOperator = flatConfig.ConditionalOperator, - IndexName = flatConfig.IndexName, - ConsistentRead = flatConfig.ConsistentRead.GetValueOrDefault(false) + FormatedExpression = ExpressionFormatConstants.In }; - // table.Scan() returns the ISearch interface but we explicitly cast it to a Search object since we rely on its internal behavior - Search scan = table.Scan(scanConfig) as Search; - return new ContextSearch(scan, flatConfig); - } + if (expr.Object is MemberExpression memberObj && expr.Arguments[0] is NewArrayExpression arrayExpr) + { + var propertyStorage = SetExpressionNameNode(storageConfig, memberObj, node, flatConfig); + foreach (var arg in arrayExpr.Expressions) + { + if (arg is not ConstantExpression constExpr) continue; - private ContextSearch ConvertScan(ContextExpression filterExpression, DynamoDBOperationConfig operationConfig) - { - DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, this.Config); - ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); + node.FormatedExpression += "#c, "; - DocumentModel.Expression expression = null; - if (filterExpression is { Filter: null }) + SetExpressionValueNode(constExpr, node, propertyStorage, flatConfig); + } + } + else { - expression = ComposeExpression(filterExpression.Filter, storageConfig, flatConfig); + throw new NotSupportedException("Expected MemberExpression with NewArrayExpression as argument for In method call."); } - Table table = GetTargetTable(storageConfig, flatConfig); - var scanConfig = new ScanOperationConfig + if (node.FormatedExpression.EndsWith(", ")) { - AttributesToGet = storageConfig.AttributesToGet, - Select = SelectValues.SpecificAttributes, - FilterExpression = expression, - IndexName = flatConfig.IndexName, - ConsistentRead = flatConfig.ConsistentRead.GetValueOrDefault(false) - }; - - // table.Scan() returns the ISearch interface but we explicitly cast it to a Search object since we rely on its internal behavior - Search scan = table.Scan(scanConfig) as Search; - return new ContextSearch(scan, flatConfig); - } - - private ContextSearch ConvertFromScan<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ScanOperationConfig scanConfig, DynamoDBOperationConfig operationConfig) - { - DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); - ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); - Table table = GetTargetTable(storageConfig, flatConfig); - - // table.Scan() returns the ISearch interface but we explicitly cast it to a Search object since we rely on its internal behavior - Search search = table.Scan(scanConfig) as Search; - return new ContextSearch(search, flatConfig); + node.FormatedExpression = node.FormatedExpression.Substring(0, node.FormatedExpression.Length - 2); + } + node.FormatedExpression += ")"; + return node; } - private ContextSearch ConvertFromQuery<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(QueryOperationConfig queryConfig, DynamoDBOperationConfig operationConfig) + private ExpressionNode HandleBetweenMethodCall(MethodCallExpression expr, + ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) { - DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); - ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); - Table table = GetTargetTable(storageConfig, flatConfig); + var node = new ExpressionNode + { + FormatedExpression = ExpressionFormatConstants.Between + }; - // table.Query() returns the ISearch interface but we explicitly cast it to a Search object since we rely on its internal behavior - Search search = table.Query(queryConfig) as Search; - return new ContextSearch(search, flatConfig); - } - private ContextSearch ConvertQueryByValue<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(object hashKeyValue, QueryOperator op, IEnumerable values, DynamoDBOperationConfig operationConfig) - { - if (operationConfig!=null) + if (expr.Arguments.Count == 3 && expr.Object == null) { - operationConfig.ValidateFilter(); - } + var collectionExpr = expr.Arguments[0] as MemberExpression; + var constExprLeft = expr.Arguments[1] as ConstantExpression; + var constExprRight = expr.Arguments[2] as ConstantExpression; - DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); - ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); - //todo - add support for expression - ContextSearch query; - if (operationConfig is { ExpressionFilter: { Filter: not null } }) - { - query = ConvertQueryByValueWithExpression(hashKeyValue, op, values, operationConfig.ExpressionFilter.Filter, operationConfig, storageConfig); + if (collectionExpr != null && constExprLeft != null && constExprRight != null) + { + var propertyStorage = SetExpressionNameNode(storageConfig, collectionExpr, node, flatConfig); + SetExpressionValueNode(constExprLeft, node, propertyStorage, flatConfig); + SetExpressionValueNode(constExprRight, node, propertyStorage, flatConfig); + } } else { - List conditions = CreateQueryConditions(flatConfig, op, values, storageConfig); - query = ConvertQueryByValue(hashKeyValue, conditions, operationConfig, storageConfig); + throw new NotSupportedException("Expected MemberExpression with NewArrayExpression as argument for In method call."); } - return query; + + return node; } - private ContextSearch ConvertQueryByValueWithExpression(object hashKeyValue, QueryOperator op, IEnumerable values, - Expression filterExpression, DynamoDBOperationConfig operationConfig, ItemStorageConfig storageConfig) + private ExpressionNode HandleStartsWithMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) { - DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); - - if (storageConfig == null) - storageConfig = StorageConfigCache.GetConfig(flatConfig); - if (operationConfig.QueryFilter != null && operationConfig.QueryFilter.Count != 0) + var node = new ExpressionNode { - throw new InvalidOperationException("QueryFilter is not supported with filter expression. Use either QueryFilter or filter expression, but not both."); + FormatedExpression = ExpressionFormatConstants.BeginsWith + }; + if (expr.Object is MemberExpression memberObj && expr.Arguments[0] is ConstantExpression argConst) + { + SetExpressionNodeAttributes(storageConfig, memberObj, argConst, node, flatConfig); + } + else + { + throw new NotSupportedException("Expected MemberExpression with ConstantExpression as argument for StartsWith method call."); } - return ConvertQueryHelper(hashKeyValue, op, values, flatConfig, storageConfig, filterExpression); + return node; } - private ContextSearch - ConvertQueryByValue<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>( - object hashKeyValue, IEnumerable conditions, DynamoDBOperationConfig operationConfig, - ItemStorageConfig storageConfig = null) + private ExpressionNode HandleContainsMethodCall(MethodCallExpression expr, + ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) { - // DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); - // if (storageConfig == null) - // storageConfig = StorageConfigCache.GetConfig(flatConfig); - - // List indexNames; - // QueryFilter filter = ComposeQueryFilter(flatConfig, hashKeyValue, conditions, storageConfig, out indexNames); - // return ConvertQueryHelper(flatConfig, storageConfig, filter, indexNames); - - if (operationConfig != null) + var node = new ExpressionNode { - operationConfig.ValidateFilter(); + FormatedExpression = ExpressionFormatConstants.Contains + }; + if (expr.Object is MemberExpression memberObj && expr.Arguments[0] is ConstantExpression argConst) + { + SetExpressionNodeAttributes(storageConfig, memberObj, argConst, node, flatConfig); } - - DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); - - if (storageConfig == null) - storageConfig = StorageConfigCache.GetConfig(flatConfig); - - ContextSearch query; - if (operationConfig is { ExpressionFilter: { Filter: not null } }) + else if (expr.Arguments.Count == 2 && expr.Object == null) { - query = ConvertQueryByValueWithExpression(hashKeyValue, QueryOperator.Equal, null, - operationConfig.ExpressionFilter.Filter, operationConfig, storageConfig); + var collectionExpr = expr.Arguments[0] as MemberExpression; + var constExpr = expr.Arguments[1] as ConstantExpression; + + if (collectionExpr != null && constExpr != null) + { + SetExpressionNodeAttributes(storageConfig, collectionExpr, constExpr, node, flatConfig); + } + else + { + throw new NotSupportedException( + "Expected MemberExpression with ConstantExpression as argument for Contains method call."); + } } else { - - List indexNames; - QueryFilter filter = ComposeQueryFilter(flatConfig, hashKeyValue, conditions, storageConfig, out indexNames); - query = ConvertQueryHelper(flatConfig, storageConfig, filter, indexNames); + throw new NotSupportedException( + "Expected MemberExpression with ConstantExpression as argument for Contains method call."); } - return query; + return node; } - private ContextSearch - ConvertQueryHelper<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>( - object hashKeyValue, QueryOperator op, IEnumerable values, DynamoDBFlatConfig flatConfig, - ItemStorageConfig storageConfig, - Expression filterExpression) + private ExpressionNode HandleEqualsMethodCall(MethodCallExpression expr, ItemStorageConfig storageConfig, + DynamoDBFlatConfig flatConfig) { - ValidateHashKey(hashKeyValue, storageConfig); - ValidateQueryKeyConfiguration(storageConfig, flatConfig); + const string formatedExpression = ExpressionFormatConstants.Equal; + var node = new ExpressionNode + { + FormatedExpression = formatedExpression + }; - var hashKeyEntry = HashKeyValueToDynamoDBEntry(flatConfig, hashKeyValue, storageConfig); - var keyExpression = new DocumentModel.Expression + if (expr.Object is MemberExpression member && + expr.Arguments[0] is ConstantExpression constant && + constant.Value == null) { - ExpressionStatement = "#hashKey = :hashKey", - ExpressionAttributeValues = new Dictionary - { - { ":hashKey", hashKeyEntry.Item2 } - }, - ExpressionAttributeNames = new Dictionary + SetExpressionNodeAttributes(storageConfig, member, constant, node, flatConfig); + return node; + } + else if (expr.Arguments.Count == 2 && expr.Object == null) + { + var memberObj = ContextExpressionsUtils.GetMember(expr.Arguments[0]) + ?? ContextExpressionsUtils.GetMember(expr.Arguments[1]); + var argConst = ContextExpressionsUtils.GetConstant(expr.Arguments[1]) + ?? ContextExpressionsUtils.GetConstant(expr.Arguments[0]); + if (memberObj != null && argConst != null) { - { "#hashKey", hashKeyEntry.Item1 } + SetExpressionNodeAttributes(storageConfig, memberObj, argConst, node, flatConfig); + return node; } - }; + } - string rangeKeyPropertyName; + throw new NotSupportedException("Expected MemberExpression with ConstantExpression as argument for Equals method call."); + } - string indexName = flatConfig.IndexName; - if (string.IsNullOrEmpty(indexName)) - rangeKeyPropertyName = storageConfig.RangeKeyPropertyNames.FirstOrDefault(); - else - rangeKeyPropertyName = storageConfig.GetRangeKeyByIndex(indexName); + private void SetExpressionNodeAttributes(ItemStorageConfig storageConfig, Expression memberObj, + ConstantExpression argConst, ExpressionNode node, DynamoDBFlatConfig flatConfig) + { + var propertyStorage = SetExpressionNameNode(storageConfig, memberObj, node, flatConfig); + SetExpressionValueNode(argConst, node, propertyStorage, flatConfig); + } - if (!string.IsNullOrEmpty(rangeKeyPropertyName) && values!=null) + private void SetExpressionValueNode(ConstantExpression argConst, ExpressionNode node, PropertyStorage propertyStorage, DynamoDBFlatConfig flatConfig) + { + DynamoDBEntry entry = ToDynamoDBEntry(propertyStorage, argConst?.Value, flatConfig, canReturnScalarInsteadOfList: true); + var valuesNode = new ExpressionNode() { - //todo implement QueryOperator to expression mapping - keyExpression.ExpressionStatement += GetRangeKeyConditionExpression($"#rangeKey", op); - keyExpression.ExpressionAttributeNames.Add("#rangeKey", rangeKeyPropertyName); - var valuesList = values?.ToList(); - if (op == QueryOperator.Between && valuesList != null && valuesList.Count() == 2) + FormatedExpression = ExpressionFormatConstants.Value + }; + valuesNode.Values.Enqueue(entry); + node.Children.Enqueue(valuesNode); + } + + private PropertyStorage ResolveNestedPropertyStorage(StorageConfig rootConfig, DynamoDBFlatConfig flatConfig, + List path, Queue namesNodeNames) + { + StorageConfig currentConfig = rootConfig; + PropertyStorage propertyStorage = null; + for (int i = 0; i < path.Count; i++) + { + var pathNode = path[i]; + + // If the path node is a map, just add the name to the queue + if (pathNode.IsMap) { - //todo - use ToDynamoDBEntry to convert values to DynamoDBEntry - keyExpression.ExpressionAttributeValues.Add(":rangeKey0", ToAttributeValue(valuesList.ElementAt(0))); - keyExpression.ExpressionAttributeValues.Add(":rangeKey1", ToAttributeValue(valuesList.ElementAt(1))); + namesNodeNames.Enqueue(pathNode.Path); + continue; } - else + + propertyStorage = currentConfig.GetPropertyStorage(pathNode.Path); + if (propertyStorage == null) + throw new InvalidOperationException($"Property '{pathNode.Path}' not found in storage config."); + // If the property is ignored, throw an exception + if (propertyStorage.IsIgnored) { - keyExpression.ExpressionAttributeValues.Add(":rangeKey0", ToAttributeValue(valuesList.FirstOrDefault())); + throw new InvalidOperationException($"Property '{pathNode.Path}' is marked as ignored and cannot be used in a filter expression."); } - } - Table table = GetTargetTable(storageConfig, flatConfig); - var queryConfig = new QueryOperationConfig - { - ConsistentRead = flatConfig.ConsistentRead.Value, - BackwardSearch = flatConfig.BackwardQuery.Value, - KeyExpression = keyExpression, - }; + namesNodeNames.Enqueue(propertyStorage.AttributeName); + // If not the last segment, descend into the nested StorageConfig + if (i >= path.Count - 1) continue; - var expression = ComposeExpression(filterExpression, storageConfig, flatConfig); + // Only descend if the property is a complex type (not primitive/string) + var propertyType = propertyStorage.MemberType; + if (Utils.IsPrimitive(propertyType)) + throw new InvalidOperationException($"Property '{pathNode.Path}' is not a complex type."); - //TODO string indexName = GetQueryIndexName(currentConfig, indexNames); - queryConfig.FilterExpression = expression; - - if (string.IsNullOrEmpty(indexName)) - { - queryConfig.Select = SelectValues.SpecificAttributes; - List attributesToGet = storageConfig.AttributesToGet; - queryConfig.AttributesToGet = attributesToGet; - } - else - { - queryConfig.IndexName = indexName; - queryConfig.Select = SelectValues.AllProjectedAttributes; - } - Search query = table.Query(queryConfig) as Search; + // Determine the element type if the property is a collection + var nextPathNode = path[i + 1]; - return new ContextSearch(query, flatConfig); - } + Type elementType = null; + var depth = pathNode.IndexDepth; + if (nextPathNode is { IsMap: true }) + { + depth += nextPathNode.IndexDepth; + } - private static string GetRangeKeyConditionExpression( string rangeKeyAlias, QueryOperator op) - { - switch (op) - { - case QueryOperator.Equal: - return $" AND {rangeKeyAlias} = :rangeKey0"; - case QueryOperator.LessThan: - return $" AND {rangeKeyAlias} < :rangeKey0"; - case QueryOperator.LessThanOrEqual: - return $" AND {rangeKeyAlias} <= :rangeKey0"; - case QueryOperator.GreaterThan: - return $" AND {rangeKeyAlias} > :rangeKey0"; - case QueryOperator.GreaterThanOrEqual: - return $" AND {rangeKeyAlias} >= :rangeKey0"; - case QueryOperator.Between: - return $" AND {rangeKeyAlias} BETWEEN :rangeKey0 AND :rangeKey0"; - case QueryOperator.BeginsWith: - return $" AND begins_with({rangeKeyAlias}, :rangeKey0)"; - default: - throw new NotSupportedException($"QueryOperator '{op}' is not supported for key conditions."); + var nodePropertyType = propertyType; + var currentDepth = 0; + + while (currentDepth <= depth && nodePropertyType != null && Utils.ImplementsInterface(nodePropertyType, typeof(ICollection<>)) + && nodePropertyType != typeof(string)) + { + elementType = Utils.GetElementType(nodePropertyType); + if (elementType == null) + { + IsSupportedDictionaryType(nodePropertyType, out elementType); + } + nodePropertyType = elementType; + currentDepth++; + } + elementType ??= propertyType; + + ItemStorageConfig config = StorageConfigCache.GetConfig(elementType, flatConfig); + currentConfig = config.BaseTypeStorageConfig; } - } + return propertyStorage; + } - private ContextSearch ConvertQueryHelper(DynamoDBFlatConfig currentConfig, ItemStorageConfig storageConfig, QueryFilter filter, List indexNames) + private PropertyStorage SetExpressionNameNode(ItemStorageConfig storageConfig, Expression memberObj, + ExpressionNode node, DynamoDBFlatConfig flatConfig) { - Table table = GetTargetTable(storageConfig, currentConfig); - string indexName = GetQueryIndexName(currentConfig, indexNames); - var queryConfig = new QueryOperationConfig - { - Filter = filter, - ConsistentRead = currentConfig.ConsistentRead.Value, - BackwardSearch = currentConfig.BackwardQuery.Value, - IndexName = indexName, - ConditionalOperator = currentConfig.ConditionalOperator - }; - if (string.IsNullOrEmpty(indexName)) + var path = ContextExpressionsUtils.ExtractPathNodes(memberObj); + if (path.Count == 0) { - queryConfig.Select = SelectValues.SpecificAttributes; - List attributesToGet = storageConfig.AttributesToGet; - queryConfig.AttributesToGet = attributesToGet; + throw new InvalidOperationException("Expected a valid property path in the expression."); } - else + var namesNode = new ExpressionNode() { - queryConfig.Select = SelectValues.AllProjectedAttributes; - } + FormatedExpression = string.Join(".", path.Select(pn => pn.FormattedPath)) + }; - // table.Query() returns the ISearch interface but we explicitly cast it to a Search object since we rely on its internal behavior - Search query = table.Query(queryConfig) as Search; + var propertyStorage = ResolveNestedPropertyStorage(storageConfig.BaseTypeStorageConfig, flatConfig, path, namesNode.Names); + node.Children.Enqueue(namesNode); - return new ContextSearch(query, currentConfig); + return propertyStorage; } - private AsyncSearch FromSearchAsync<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ContextSearch contextSearch) - { - return new AsyncSearch(this, contextSearch); - } #endregion - } } diff --git a/sdk/src/Services/DynamoDBv2/Custom/DocumentModel/ExpressionBuilder.cs b/sdk/src/Services/DynamoDBv2/Custom/DocumentModel/ExpressionBuilder.cs index bbc20dd329b3..3edd1f69bef2 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DocumentModel/ExpressionBuilder.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DocumentModel/ExpressionBuilder.cs @@ -646,7 +646,7 @@ private Queue BuildChildNodes() private ExpressionNode NotBuildCondition(ExpressionNode node) { - node.FormatedExpression = "NOT (#c)"; + node.FormatedExpression = ExpressionFormatConstants.Not; return node; } @@ -670,27 +670,27 @@ private ExpressionNode CompareBuildCondition(ConditionMode conditionMode, Expres { case ConditionMode.Equal: node.FormatedExpression = - $"#c = #c"; + ExpressionFormatConstants.Equal; break; case ConditionMode.NotEqual: node.FormatedExpression = - $"#c <> #c"; + ExpressionFormatConstants.NotEqual; break; case ConditionMode.LessThan: node.FormatedExpression = - $"#c < #c"; + ExpressionFormatConstants.LessThan; break; case ConditionMode.LessThanOrEqual: node.FormatedExpression = - $"#c <= #c"; + ExpressionFormatConstants.LessThanOrEqual; break; case ConditionMode.GreaterThan: node.FormatedExpression = - $"#c > #c"; + ExpressionFormatConstants.GreaterThan; break; case ConditionMode.GreaterThanOrEqual: node.FormatedExpression = - $"#c >= #c"; + ExpressionFormatConstants.GreaterThanOrEqual; break; default: throw new InvalidOperationException($"Unsupported mode: {conditionMode}"); @@ -701,37 +701,37 @@ private ExpressionNode CompareBuildCondition(ConditionMode conditionMode, Expres private ExpressionNode ContainsBuildCondition(ExpressionNode node) { - node.FormatedExpression = "contains (#c, #c)"; + node.FormatedExpression = ExpressionFormatConstants.Contains; return node; } private ExpressionNode BeginsWithBuildCondition(ExpressionNode node) { - node.FormatedExpression = "begins_with (#c, #c)"; + node.FormatedExpression = ExpressionFormatConstants.BeginsWith; return node; } private ExpressionNode AttributeTypeBuildCondition(ExpressionNode node) { - node.FormatedExpression = "attribute_type (#c, #c)"; + node.FormatedExpression = ExpressionFormatConstants.AttributeType; return node; } private ExpressionNode AttributeNotExistsBuildCondition(ExpressionNode node) { - node.FormatedExpression = "attribute_not_exists (#c)"; + node.FormatedExpression = ExpressionFormatConstants.AttributeNotExists; return node; } private ExpressionNode AttributeExistsBuildCondition(ExpressionNode node) { - node.FormatedExpression = "attribute_exists (#c)"; + node.FormatedExpression = ExpressionFormatConstants.AttributeExists; return node; } private ExpressionNode InBuildCondition(ConditionExpressionBuilder conditionBuilder, ExpressionNode node) { - node.FormatedExpression = "#c IN ("; + node.FormatedExpression = ExpressionFormatConstants.In; for(int i = 1; i < node.Children.Count; i++){ node.FormatedExpression += "#c, "; @@ -747,7 +747,7 @@ private ExpressionNode InBuildCondition(ConditionExpressionBuilder conditionBuil private ExpressionNode BetweenBuildCondition(ExpressionNode node) { - node.FormatedExpression = "#c BETWEEN #c AND #c"; + node.FormatedExpression = ExpressionFormatConstants.Between; return node; } @@ -1184,7 +1184,7 @@ internal override ExpressionNode Build() return new ExpressionNode { Values = values, - FormatedExpression = "#v" + FormatedExpression = ExpressionFormatConstants.Value }; } } @@ -1487,10 +1487,10 @@ internal override ExpressionNode Build() node.FormatedExpression = _mode switch { - SetValueMode.Plus => "#c + #c", - SetValueMode.Minus => "#c - #c", - SetValueMode.ListAppend=> "list_append(#c, #c)", - SetValueMode.IfNotExists => "if_not_exists(#c, #c)", + SetValueMode.Plus => ExpressionFormatConstants.Plus, + SetValueMode.Minus => ExpressionFormatConstants.Minus, + SetValueMode.ListAppend=> ExpressionFormatConstants.ListAppend, + SetValueMode.IfNotExists => ExpressionFormatConstants.IfNotExists, _ => throw new InvalidOperationException($"Unsupported SetValueMode: '{_mode}'.") }; @@ -1614,4 +1614,31 @@ internal class KeyAttributeAliasList /// public List ValuesList { get; set; } = new(); } + + + /// + /// Contains constants for formatted DynamoDB expression templates. + /// + internal static class ExpressionFormatConstants + { + public const string Equal = "#c = #c"; + public const string NotEqual = "#c <> #c"; + public const string LessThan = "#c < #c"; + public const string LessThanOrEqual = "#c <= #c"; + public const string GreaterThan = "#c > #c"; + public const string GreaterThanOrEqual = "#c >= #c"; + public const string AttributeType = "attribute_type (#c, #c)"; + public const string AttributeNotExists = "attribute_not_exists (#c)"; + public const string AttributeExists = "attribute_exists (#c)"; + public const string In = "#c IN ("; + public const string Between = "#c BETWEEN #c AND #c"; + public const string BeginsWith = "begins_with (#c, #c)"; + public const string Contains = "contains (#c, #c)"; + public const string Not = "NOT (#c)"; + public const string Value = "#v"; + public const string Plus = "#c + #c"; + public const string Minus = "#c - #c"; + public const string ListAppend = "list_append(#c, #c)"; + public const string IfNotExists = "if_not_exists(#c, #c)"; + } } \ No newline at end of file From c59a0a9d84ec5600a9c03e787ed66c03144f9f92 Mon Sep 17 00:00:00 2001 From: irina-herciu Date: Wed, 11 Jun 2025 12:15:10 +0300 Subject: [PATCH 04/13] unit tests --- .../Custom/DataModel/ContextExpression.cs | 1 - .../Custom/DataModel/ContextInternal.cs | 63 ++++----- .../Custom/ContextExpressionsUtilsTests.cs | 126 ++++++++++++++++++ 3 files changed, 153 insertions(+), 37 deletions(-) create mode 100644 sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs index cd533f362ffc..271645489ba8 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs @@ -127,7 +127,6 @@ internal static bool IsMember(Expression expr) }; } - internal static ConstantExpression GetConstant(Expression expr) { return expr switch diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs index 4f6cb7e5d9af..1c0a25e13a8e 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs @@ -13,7 +13,7 @@ * permissions and limitations under the License. */ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Linq; @@ -148,7 +148,7 @@ internal Table GetTargetTable(ItemStorageConfig storageConfig, DynamoDBFlatConfi return table; } -// This is the call we want to avoid with disableFetchingTableMetadata = true, but as long as we still support false, we still need to call the discouraged sync-over-async 'Table.LoadTable(Client, emptyConfig)' + // This is the call we want to avoid with disableFetchingTableMetadata = true, but as long as we still support false, we still need to call the discouraged sync-over-async 'Table.LoadTable(Client, emptyConfig)' #pragma warning disable CS0618 // Retrieves Config-less Table from cache or constructs it on cache-miss @@ -157,7 +157,7 @@ internal Table GetTargetTable(ItemStorageConfig storageConfig, DynamoDBFlatConfi internal Table GetUnconfiguredTable(string tableName, bool disableFetchingTableMetadata = false) { Table table; - + try { _readerWriterLockSlim.EnterReadLock(); @@ -169,7 +169,7 @@ internal Table GetUnconfiguredTable(string tableName, bool disableFetchingTableM } finally { - if(_readerWriterLockSlim.IsReadLockHeld) + if (_readerWriterLockSlim.IsReadLockHeld) { _readerWriterLockSlim.ExitReadLock(); } @@ -178,19 +178,19 @@ internal Table GetUnconfiguredTable(string tableName, bool disableFetchingTableM try { _readerWriterLockSlim.EnterWriteLock(); - + // Check to see if another thread got the write lock before this thread and filled the cache. if (tablesMap.TryGetValue(tableName, out table)) { return table; } - + if (disableFetchingTableMetadata) { return null; } - + var emptyConfig = new TableConfig(tableName, conversion: null, consumer: Table.DynamoDBConsumer.DataModel, storeAsEpoch: null, storeAsEpochLong: null, isEmptyStringValueEnabled: false, metadataCachingMode: Config.MetadataCachingMode); table = Table.LoadTable(Client, emptyConfig) as Table; @@ -200,7 +200,7 @@ internal Table GetUnconfiguredTable(string tableName, bool disableFetchingTableM } finally { - if(_readerWriterLockSlim.IsWriteLockHeld) + if (_readerWriterLockSlim.IsWriteLockHeld) { _readerWriterLockSlim.ExitWriteLock(); } @@ -281,14 +281,14 @@ private static void ValidateConfigAgainstTable(ItemStorageConfig config, Table t private static void CompareKeys(ItemStorageConfig config, Table table, List attributes, List properties, string keyType) { if (attributes.Count != properties.Count) - throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, + throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "Number of {0} keys on table {1} does not match number of hash keys on type {2}", keyType, table.TableName, config.BaseTypeStorageConfig.TargetType.FullName)); foreach (string hashProperty in properties) { PropertyStorage property = config.BaseTypeStorageConfig.GetPropertyStorage(hashProperty); if (!attributes.Contains(property.AttributeName)) - throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, + throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "Key property {0} on type {1} does not correspond to a {2} key on table {3}", hashProperty, config.BaseTypeStorageConfig.TargetType.FullName, keyType, table.TableName)); } @@ -512,7 +512,7 @@ private object FromDynamoDBEntry(SimplePropertyStorage propertyStorage, DynamoDB var conversion = flatConfig.Conversion; var targetType = propertyStorage.MemberType; - + if (conversion.HasConverter(targetType)) { var output = conversion.ConvertFromEntry(targetType, entry); @@ -622,14 +622,14 @@ private bool TryFromListToArray([DynamicallyAccessedMembers(InternalConstants.Da } var elementType = Utils.GetElementType(targetType); - var array = (Array)Utils.InstantiateArray(targetType,list.Entries.Count); + var array = (Array)Utils.InstantiateArray(targetType, list.Entries.Count); var propertyStorage = new SimplePropertyStorage(elementType, parentPropertyStorage); for (int i = 0; i < list.Entries.Count; i++) { var entry = list.Entries[i]; var item = FromDynamoDBEntry(propertyStorage, entry, flatConfig); - array.SetValue(item,i); + array.SetValue(item, i); } output = array; @@ -915,7 +915,7 @@ private static bool TryGetValue(object instance, MemberInfo member, out object v { FieldInfo fieldInfo = member as FieldInfo; PropertyInfo propertyInfo = member as PropertyInfo; - + if (fieldInfo != null) { value = fieldInfo.GetValue(instance); @@ -978,7 +978,7 @@ private QueryFilter ComposeQueryFilter(DynamoDBFlatConfig currentConfig, object return ComposeQueryFilterHelper(currentConfig, hashKey, conditions, storageConfig, out indexNames); } - private (string,DynamoDBEntry) HashKeyValueToDynamoDBEntry(DynamoDBFlatConfig currentConfig, object hashKeyValue, + private (string, DynamoDBEntry) HashKeyValueToDynamoDBEntry(DynamoDBFlatConfig currentConfig, object hashKeyValue, ItemStorageConfig storageConfig) { // Set hash key property name @@ -992,7 +992,7 @@ private QueryFilter ComposeQueryFilter(DynamoDBFlatConfig currentConfig, object DynamoDBEntry hashKeyEntry = ValueToDynamoDBEntry(propertyStorage, hashKeyValue, currentConfig); if (hashKeyEntry == null) throw new InvalidOperationException("Unable to convert hash key value for property " + hashKeyProperty); - return (hashAttributeName,hashKeyEntry); + return (hashAttributeName, hashKeyEntry); } private static void ValidateHashKey(object hashKeyValue, ItemStorageConfig storageConfig) @@ -1039,7 +1039,7 @@ private QueryFilter ComposeQueryFilterHelper( throw new ArgumentNullException("hashKey"); ValidateQueryKeyConfiguration(storageConfig, currentConfig); - + QueryFilter filter = new QueryFilter(); // Configure hash-key equality condition @@ -1301,7 +1301,7 @@ public ContextSearch(Search search, DynamoDBFlatConfig flatConfig) } - private ContextSearch ConvertScan(ContextExpression filterExpression, DynamoDBOperationConfig operationConfig) + private ContextSearch ConvertScan<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ContextExpression filterExpression, DynamoDBOperationConfig operationConfig) { DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, this.Config); ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); @@ -1351,7 +1351,7 @@ private ContextSearch ConvertScan(ContextExpression filterExpression, DynamoD private ContextSearch ConvertQueryByValue<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(object hashKeyValue, QueryOperator op, IEnumerable values, DynamoDBOperationConfig operationConfig) { - if (operationConfig!=null) + if (operationConfig != null) { operationConfig.ValidateFilter(); } @@ -1372,7 +1372,7 @@ private ContextSearch ConvertScan(ContextExpression filterExpression, DynamoD return query; } - private ContextSearch ConvertQueryByValueWithExpression(object hashKeyValue, QueryOperator op, IEnumerable values, + private ContextSearch ConvertQueryByValueWithExpression<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(object hashKeyValue, QueryOperator op, IEnumerable values, Expression filterExpression, DynamoDBOperationConfig operationConfig, ItemStorageConfig storageConfig) { DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); @@ -1392,14 +1392,6 @@ private ContextSearch object hashKeyValue, IEnumerable conditions, DynamoDBOperationConfig operationConfig, ItemStorageConfig storageConfig = null) { - // DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); - // if (storageConfig == null) - // storageConfig = StorageConfigCache.GetConfig(flatConfig); - - // List indexNames; - // QueryFilter filter = ComposeQueryFilter(flatConfig, hashKeyValue, conditions, storageConfig, out indexNames); - // return ConvertQueryHelper(flatConfig, storageConfig, filter, indexNames); - if (operationConfig != null) { operationConfig.ValidateFilter(); @@ -1407,13 +1399,12 @@ private ContextSearch DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); - if (storageConfig == null) - storageConfig = StorageConfigCache.GetConfig(flatConfig); - + storageConfig ??= StorageConfigCache.GetConfig(flatConfig); + ContextSearch query; if (operationConfig is { ExpressionFilter: { Filter: not null } }) { - query = ConvertQueryByValueWithExpression(hashKeyValue, QueryOperator.Equal, null, + query = ConvertQueryByValueWithExpression(hashKeyValue, QueryOperator.Equal, null, operationConfig.ExpressionFilter.Filter, operationConfig, storageConfig); } else @@ -1457,7 +1448,7 @@ private ContextSearch else rangeKeyPropertyName = storageConfig.GetRangeKeyByIndex(indexName); - if (!string.IsNullOrEmpty(rangeKeyPropertyName) && values!=null) + if (!string.IsNullOrEmpty(rangeKeyPropertyName) && values != null) { //todo implement QueryOperator to expression mapping keyExpression.ExpressionStatement += ContextExpressionsUtils.GetRangeKeyConditionExpression($"#rangeKey", op); @@ -1500,7 +1491,7 @@ private ContextSearch //TODO string indexName = GetQueryIndexName(currentConfig, indexNames); queryConfig.FilterExpression = expression; - + if (string.IsNullOrEmpty(indexName)) { queryConfig.Select = SelectValues.SpecificAttributes; @@ -1905,9 +1896,9 @@ expr.Arguments[0] is ConstantExpression constant && } else if (expr.Arguments.Count == 2 && expr.Object == null) { - var memberObj = ContextExpressionsUtils.GetMember(expr.Arguments[0]) + var memberObj = ContextExpressionsUtils.GetMember(expr.Arguments[0]) ?? ContextExpressionsUtils.GetMember(expr.Arguments[1]); - var argConst = ContextExpressionsUtils.GetConstant(expr.Arguments[1]) + var argConst = ContextExpressionsUtils.GetConstant(expr.Arguments[1]) ?? ContextExpressionsUtils.GetConstant(expr.Arguments[0]); if (memberObj != null && argConst != null) { diff --git a/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs new file mode 100644 index 000000000000..7b96704e760e --- /dev/null +++ b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs @@ -0,0 +1,126 @@ +using Amazon.DynamoDBv2.DataModel; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using Amazon.DynamoDBv2.DocumentModel; +using Expression = System.Linq.Expressions.Expression; + +namespace AWSSDK_DotNet.UnitTests +{ + [TestClass] + public class ContextExpressionsUtilsTests + { + [TestMethod] + [DataRow(QueryOperator.Equal, " AND myKey = :rangeKey0")] + [DataRow(QueryOperator.LessThan, " AND myKey < :rangeKey0")] + [DataRow(QueryOperator.LessThanOrEqual, " AND myKey <= :rangeKey0")] + [DataRow(QueryOperator.GreaterThan, " AND myKey > :rangeKey0")] + [DataRow(QueryOperator.GreaterThanOrEqual, " AND myKey >= :rangeKey0")] + [DataRow(QueryOperator.Between, " AND myKey BETWEEN :rangeKey0 AND :rangeKey0")] + [DataRow(QueryOperator.BeginsWith, " AND begins_with(myKey, :rangeKey0)")] + public void GetRangeKeyConditionExpression_ReturnsExpected(QueryOperator op, string expected) + { + var result = ContextExpressionsUtils.GetRangeKeyConditionExpression("myKey", op); + Assert.AreEqual(expected, result); + } + + [TestMethod] + public void GetRangeKeyConditionExpression_ThrowsOnUnsupported() + { + Assert.ThrowsException(() => + ContextExpressionsUtils.GetRangeKeyConditionExpression("myKey", (QueryOperator)999)); + } + + [TestMethod] + public void IsMember_ReturnsTrueForMemberExpression() + { + Expression> expr = () => "".Length; + Assert.IsTrue(ContextExpressionsUtils.IsMember(expr.Body)); + } + + [TestMethod] + public void IsMember_ReturnsTrueForUnaryMemberExpression() + { + Expression> expr = () => (object)"".Length; + Assert.IsTrue(ContextExpressionsUtils.IsMember(expr.Body)); + } + + [TestMethod] + public void IsMember_ReturnsFalseForOtherExpressions() + { + Expression> expr = () => 5 + 3; + Assert.IsFalse(ContextExpressionsUtils.IsMember(expr.Body)); + } + + [TestMethod] + public void GetConstant_ReturnsConstantExpression() + { + var constExpr = Expression.Constant(42); + var result = ContextExpressionsUtils.GetConstant(constExpr); + Assert.AreEqual(constExpr, result); + } + + [TestMethod] + public void GetConstant_ReturnsConstantFromUnary() + { + var constExpr = Expression.Constant(42); + var unaryExpr = Expression.Convert(constExpr, typeof(object)); + var result = ContextExpressionsUtils.GetConstant(unaryExpr); + Assert.AreEqual(constExpr, result); + } + + [TestMethod] + public void GetConstant_ReturnsNullForUnsupported() + { + var paramExpr = Expression.Parameter(typeof(int), "x"); + Assert.IsNull(ContextExpressionsUtils.GetConstant(paramExpr)); + } + + [TestMethod] + public void IsComparison_ReturnsTrueForComparisonTypes() + { + Assert.IsTrue(ContextExpressionsUtils.IsComparison(ExpressionType.Equal)); + Assert.IsTrue(ContextExpressionsUtils.IsComparison(ExpressionType.NotEqual)); + Assert.IsTrue(ContextExpressionsUtils.IsComparison(ExpressionType.GreaterThan)); + Assert.IsTrue(ContextExpressionsUtils.IsComparison(ExpressionType.GreaterThanOrEqual)); + Assert.IsTrue(ContextExpressionsUtils.IsComparison(ExpressionType.LessThan)); + Assert.IsTrue(ContextExpressionsUtils.IsComparison(ExpressionType.LessThanOrEqual)); + } + + [TestMethod] + public void IsComparison_ReturnsFalseForOtherTypes() + { + Assert.IsFalse(ContextExpressionsUtils.IsComparison(ExpressionType.Add)); + } + + [TestMethod] + public void GetMember_ReturnsMemberExpression() + { + Expression> expr = () => "".Length; + var result = ContextExpressionsUtils.GetMember(expr.Body); + Assert.IsNotNull(result); + Assert.AreEqual("Length", result.Member.Name); + } + + [TestMethod] + public void GetMember_ReturnsNullForNonMember() + { + Expression> expr = () => 5 + 3; + Assert.IsNull(ContextExpressionsUtils.GetMember(expr.Body)); + } + + [TestMethod] + public void ExtractPathNodes_PropertyPath() + { + Expression> expr = d => d.Child.Value; + var nodes = ContextExpressionsUtils.ExtractPathNodes(expr.Body); + Assert.AreEqual(2, nodes.Count); + Assert.AreEqual("Child", nodes[0].Path); + Assert.AreEqual("Value", nodes[1].Path); + } + + class Dummy { public Dummy Child { get; set; } public int Value { get; set; } } + } +} \ No newline at end of file From 66bd10adb7d3629b07acbf0b1ab1912cf124d3d2 Mon Sep 17 00:00:00 2001 From: irina-herciu Date: Wed, 11 Jun 2025 19:00:56 +0300 Subject: [PATCH 05/13] unit tests on context internal --- .../Custom/DataModel/ContextInternal.cs | 14 +- .../Custom/ContextExpressionsUtilsTests.cs | 81 ++++++- .../UnitTests/Custom/ContextInternalTests.cs | 216 ++++++++++++++++++ 3 files changed, 294 insertions(+), 17 deletions(-) create mode 100644 sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextInternalTests.cs diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs index 1ae9ac80d934..27b6b5ca1901 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs @@ -122,10 +122,10 @@ internal static DocumentModel.Expression CreateConditionExpressionForVersion(Ite #region Atomic counters - internal static Expression BuildCounterConditionExpression(ItemStorage storage) + internal static DocumentModel.Expression BuildCounterConditionExpression(ItemStorage storage) { var atomicCounters = GetCounterProperties(storage); - Expression counterConditionExpression = null; + DocumentModel.Expression counterConditionExpression = null; if (atomicCounters.Length != 0) { @@ -143,11 +143,11 @@ private static PropertyStorage[] GetCounterProperties(ItemStorage storage) return counterProperties; } - private static Expression CreateUpdateExpressionForCounterProperties(PropertyStorage[] counterPropertyStorages) + private static DocumentModel.Expression CreateUpdateExpressionForCounterProperties(PropertyStorage[] counterPropertyStorages) { if (counterPropertyStorages.Length == 0) return null; - Expression updateExpression = new Expression(); + DocumentModel.Expression updateExpression = new DocumentModel.Expression(); var asserts = string.Empty; foreach (var propertyStorage in counterPropertyStorages) @@ -1397,13 +1397,13 @@ public ContextSearch(Search search, DynamoDBFlatConfig flatConfig) } - private ContextSearch ConvertScan<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ContextExpression filterExpression, DynamoDBOperationConfig operationConfig) + internal ContextSearch ConvertScan<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ContextExpression filterExpression, DynamoDBOperationConfig operationConfig) { DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, this.Config); ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); DocumentModel.Expression expression = null; - if (filterExpression is { Filter: null }) + if (filterExpression is not { Filter: null }) { expression = ComposeExpression(filterExpression.Filter, storageConfig, flatConfig); } @@ -1483,7 +1483,7 @@ public ContextSearch(Search search, DynamoDBFlatConfig flatConfig) } - private ContextSearch + internal ContextSearch ConvertQueryByValue<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>( object hashKeyValue, IEnumerable conditions, DynamoDBOperationConfig operationConfig, ItemStorageConfig storageConfig = null) diff --git a/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs index 7b96704e760e..ec4b00c2f2fe 100644 --- a/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs +++ b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs @@ -36,7 +36,7 @@ public void GetRangeKeyConditionExpression_ThrowsOnUnsupported() [TestMethod] public void IsMember_ReturnsTrueForMemberExpression() { - Expression> expr = () => "".Length; + Expression> expr = () => "".Length; Assert.IsTrue(ContextExpressionsUtils.IsMember(expr.Body)); } @@ -95,15 +95,6 @@ public void IsComparison_ReturnsFalseForOtherTypes() Assert.IsFalse(ContextExpressionsUtils.IsComparison(ExpressionType.Add)); } - [TestMethod] - public void GetMember_ReturnsMemberExpression() - { - Expression> expr = () => "".Length; - var result = ContextExpressionsUtils.GetMember(expr.Body); - Assert.IsNotNull(result); - Assert.AreEqual("Length", result.Member.Name); - } - [TestMethod] public void GetMember_ReturnsNullForNonMember() { @@ -121,6 +112,76 @@ public void ExtractPathNodes_PropertyPath() Assert.AreEqual("Value", nodes[1].Path); } + [TestMethod] + public void ExtractPathNodes_ListIndexer() + { + Expression> expr = d => d.Children[2].Value; + var nodes = ContextExpressionsUtils.ExtractPathNodes(expr.Body); + Assert.AreEqual(2, nodes.Count); + Assert.AreEqual("Children", nodes[0].Path); + Assert.AreEqual(1, nodes[0].IndexDepth); + Assert.AreEqual("Value", nodes[1].Path); + } + + [TestMethod] + public void ExtractPathNodes_NestedListIndexer() + { + Expression> expr = d => d.Children[1].Child.Value; + var nodes = ContextExpressionsUtils.ExtractPathNodes(expr.Body); + Assert.AreEqual(3, nodes.Count); + Assert.AreEqual("Children", nodes[0].Path); + Assert.AreEqual(1, nodes[0].IndexDepth); + Assert.AreEqual("Child", nodes[1].Path); + Assert.AreEqual("Value", nodes[2].Path); + } + + [TestMethod] + public void ExtractPathNodes_DictionaryStringIndexer() + { + Expression> expr = d => d.Map["foo"].Value; + var nodes = ContextExpressionsUtils.ExtractPathNodes(expr.Body); + Assert.AreEqual(3, nodes.Count); + Assert.AreEqual("Map", nodes[0].Path); + Assert.AreEqual("foo", nodes[1].Path); + Assert.IsTrue(nodes[1].IsMap); + Assert.AreEqual("Value", nodes[2].Path); + } + + [TestMethod] + [ExpectedException(typeof(NotSupportedException))] + public void ExtractPathNodes_UnsupportedMethodCall_Throws() + { + Expression, int>> expr = l => l.Sum(); + ContextExpressionsUtils.ExtractPathNodes(expr.Body); + } + + [TestMethod] + public void ExtractPathNodes_ConversionExpression() + { + Expression> expr = d => (object)d.Value; + var nodes = ContextExpressionsUtils.ExtractPathNodes(expr.Body); + Assert.AreEqual(1, nodes.Count); + Assert.AreEqual("Value", nodes[0].Path); + } + + [TestMethod] + public void ExtractPathNodes_FirstOrDefault() + { + Expression> expr = d => d.Children.FirstOrDefault().Value; + var nodes = ContextExpressionsUtils.ExtractPathNodes(expr.Body); + Assert.AreEqual(2, nodes.Count); + Assert.AreEqual("Children", nodes[0].Path); + Assert.AreEqual(1, nodes[0].IndexDepth); + Assert.AreEqual("Value", nodes[1].Path); + } + class Dummy { public Dummy Child { get; set; } public int Value { get; set; } } + + class ComplexDummy + { + public List Children { get; set; } + public Dictionary Map { get; set; } + public Dummy Child { get; set; } + } } } \ No newline at end of file diff --git a/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextInternalTests.cs b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextInternalTests.cs new file mode 100644 index 000000000000..f5d25ef536de --- /dev/null +++ b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextInternalTests.cs @@ -0,0 +1,216 @@ +using Amazon.DynamoDBv2; +using Amazon.DynamoDBv2.DataModel; +using Amazon.DynamoDBv2.DocumentModel; +using Amazon.DynamoDBv2.Model; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using System; +using System.Linq.Expressions; +using System.Reflection; +using System.Runtime.Remoting.Contexts; + +namespace AWSSDK_DotNet.UnitTests +{ + [TestClass] + public class ContextInternalTests + { + public class TestEntity + { + [DynamoDBHashKey] + public int Id { get; set; } + [DynamoDBRangeKey] + public string Name { get; set; } + } + + private Mock mockClient; + private DynamoDBContext context; + + [TestInitialize] + public void TestInitialize() + { + mockClient = new Mock(MockBehavior.Strict); + mockClient.Setup(m => m.Config).Returns(new AmazonDynamoDBConfig()); + mockClient.Setup(m => m.DescribeTable(It.IsAny())) + .Returns(new DescribeTableResponse + { + Table = new TableDescription + { + TableName = "TestEntity", + KeySchema = new System.Collections.Generic.List + { + new KeySchemaElement + { + AttributeName = "Id", + KeyType = KeyType.HASH + }, + new KeySchemaElement + { + AttributeName = "Name", + KeyType = KeyType.RANGE + } + }, + AttributeDefinitions = new System.Collections.Generic.List + { + new AttributeDefinition + { + AttributeName = "Id", + AttributeType = ScalarAttributeType.N + }, + new AttributeDefinition + { + AttributeName = "Name", + AttributeType = ScalarAttributeType.S + } + } + } + }); + + context = new DynamoDBContext(mockClient.Object, new DynamoDBContextConfig()); + } + + + [TestMethod] + public void ConvertScan_WithFilterExpression_ReturnsMappedFilterExpression() + { + // Create a filter expression (e => e.Id == 1) + Expression> expr = e => e.Id == 1; + var filterExpr = new ContextExpression(); + filterExpr.SetFilter(expr); + + var result = context.ConvertScan(filterExpr, null); + + Assert.IsNotNull(result); + Assert.IsNotNull(result.Search.FilterExpression); + var actualFilterExpression = result.Search.FilterExpression; + Assert.AreEqual("#C0 = :C0", actualFilterExpression.ExpressionStatement); + Assert.IsNotNull(actualFilterExpression.ExpressionAttributeNames); + Assert.IsTrue(actualFilterExpression.ExpressionAttributeNames.ContainsKey("#C0")); + Assert.AreEqual("Id", actualFilterExpression.ExpressionAttributeNames["#C0"]); + + Assert.IsNotNull(actualFilterExpression.ExpressionAttributeValues); + Assert.IsTrue(actualFilterExpression.ExpressionAttributeValues.ContainsKey(":C0")); + Assert.AreEqual(1, actualFilterExpression.ExpressionAttributeValues[":C0"].AsInt()); + } + + [TestMethod] + public void ConvertScan_WithNameFilterExpression_ReturnsMappedFilterExpression() + { + // Filter: e => e.Name == "foo" + Expression> expr = e => e.Name == "foo"; + var filterExpr = new ContextExpression(); + filterExpr.SetFilter(expr); + + var result = context.ConvertScan(filterExpr, null); + + Assert.IsNotNull(result); + Assert.IsNotNull(result.Search.FilterExpression); + var actualFilterExpression = result.Search.FilterExpression; + Assert.AreEqual("#C0 = :C0", actualFilterExpression.ExpressionStatement); + Assert.IsNotNull(actualFilterExpression.ExpressionAttributeNames); + Assert.IsTrue(actualFilterExpression.ExpressionAttributeNames.ContainsKey("#C0")); + Assert.AreEqual("Name", actualFilterExpression.ExpressionAttributeNames["#C0"]); + Assert.IsNotNull(actualFilterExpression.ExpressionAttributeValues); + Assert.IsTrue(actualFilterExpression.ExpressionAttributeValues.ContainsKey(":C0")); + Assert.AreEqual("foo", actualFilterExpression.ExpressionAttributeValues[":C0"].AsString()); + } + + [TestMethod] + public void ConvertScan_WithGreaterThanFilterExpression_ReturnsMappedFilterExpression() + { + // Filter: e => e.Id > 10 + Expression> expr = e => e.Id > 10; + var filterExpr = new ContextExpression(); + filterExpr.SetFilter(expr); + + var result = context.ConvertScan(filterExpr, null); + + Assert.IsNotNull(result); + Assert.IsNotNull(result.Search.FilterExpression); + var actualFilterExpression = result.Search.FilterExpression; + Assert.AreEqual("#C0 > :C0", actualFilterExpression.ExpressionStatement); + Assert.IsNotNull(actualFilterExpression.ExpressionAttributeNames); + Assert.IsTrue(actualFilterExpression.ExpressionAttributeNames.ContainsKey("#C0")); + Assert.AreEqual("Id", actualFilterExpression.ExpressionAttributeNames["#C0"]); + Assert.IsNotNull(actualFilterExpression.ExpressionAttributeValues); + Assert.IsTrue(actualFilterExpression.ExpressionAttributeValues.ContainsKey(":C0")); + Assert.AreEqual(10, actualFilterExpression.ExpressionAttributeValues[":C0"].AsInt()); + } + + [TestMethod] + public void ConvertScan_WithAndFilterExpression_ReturnsMappedFilterExpression() + { + // Filter: e => e.Id == 1 && e.Name == "foo" + Expression> expr = e => e.Id == 1 && e.Name == "foo"; + var filterExpr = new ContextExpression(); + filterExpr.SetFilter(expr); + + var result = context.ConvertScan(filterExpr, null); + + Assert.IsNotNull(result); + Assert.IsNotNull(result.Search.FilterExpression); + var actualFilterExpression = result.Search.FilterExpression; + Assert.AreEqual("(#C0 = :C0) AND (#C1 = :C1)", actualFilterExpression.ExpressionStatement); + Assert.IsNotNull(actualFilterExpression.ExpressionAttributeNames); + Assert.AreEqual("Id", actualFilterExpression.ExpressionAttributeNames["#C0"]); + Assert.AreEqual("Name", actualFilterExpression.ExpressionAttributeNames["#C1"]); + Assert.IsNotNull(actualFilterExpression.ExpressionAttributeValues); + Assert.AreEqual(1, actualFilterExpression.ExpressionAttributeValues[":C0"].AsInt()); + Assert.AreEqual("foo", actualFilterExpression.ExpressionAttributeValues[":C1"].AsString()); + } + + [TestMethod] + public void ConvertQueryByValue_WithHashKeyOnly() + { + // Act + var result = context.ConvertQueryByValue(1, null, null); + + // Assert + Assert.IsNotNull(result); + Assert.IsNotNull(result.Search); + var actualResult = result.Search; + Assert.IsNotNull(actualResult.Filter); + Assert.AreEqual(1,actualResult.Filter.ToConditions().Count); + Assert.IsNull(actualResult.FilterExpression); + Assert.IsNotNull(actualResult.AttributesToGet); + Assert.AreEqual(2,actualResult.AttributesToGet.Count); + } + + [TestMethod] + public void ConvertQueryByValue_WithHashKeyAndExpressionFilter() + { + // Arrange + Expression> expr = e => e.Name == "bar"; + var filterExpr = new ContextExpression(); + filterExpr.SetFilter(expr); + + var operationConfig = new DynamoDBOperationConfig + { + ExpressionFilter = filterExpr + }; + + // Act + var result = context.ConvertQueryByValue(1, null, operationConfig); + + // Assert + Assert.IsNotNull(result); + + var search = (dynamic)result; + Assert.IsNotNull(search.Search); + Assert.IsNotNull(search.Search.KeyExpression); + Assert.IsNotNull(search.Search.KeyExpression.ExpressionStatement); + Assert.IsTrue(search.Search.KeyExpression.ExpressionStatement.Contains("#hashKey = :hashKey")); + Assert.IsNotNull(search.Search.KeyExpression.ExpressionAttributeNames); + Assert.IsTrue(search.Search.KeyExpression.ExpressionAttributeNames.ContainsKey("#hashKey")); + Assert.IsNotNull(search.Search.KeyExpression.ExpressionAttributeValues); + Assert.IsTrue(search.Search.KeyExpression.ExpressionAttributeValues.ContainsKey(":hashKey")); + var keyValue = search.Search.KeyExpression.ExpressionAttributeValues[":hashKey"]; + Assert.AreEqual(1, keyValue.AsInt()); + + // Assert filter expression + Assert.IsNotNull(search.Search.FilterExpression); + Assert.AreEqual("#C0 = :C0", search.Search.FilterExpression.ExpressionStatement); + Assert.AreEqual("Name", search.Search.FilterExpression.ExpressionAttributeNames["#C0"]); + Assert.AreEqual("bar", search.Search.FilterExpression.ExpressionAttributeValues[":C0"].ToString()); + } + } +} \ No newline at end of file From 9e07b49e399a8683103a252e633b7d7a9a4c8dd8 Mon Sep 17 00:00:00 2001 From: irina-herciu Date: Wed, 11 Jun 2025 19:11:21 +0300 Subject: [PATCH 06/13] add changeLog Messages --- .../3d369d65-77a9-4b2d-a0b0-ac6cf5ff384c.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 generator/.DevConfigs/3d369d65-77a9-4b2d-a0b0-ac6cf5ff384c.json diff --git a/generator/.DevConfigs/3d369d65-77a9-4b2d-a0b0-ac6cf5ff384c.json b/generator/.DevConfigs/3d369d65-77a9-4b2d-a0b0-ac6cf5ff384c.json new file mode 100644 index 000000000000..3c5459f27dbc --- /dev/null +++ b/generator/.DevConfigs/3d369d65-77a9-4b2d-a0b0-ac6cf5ff384c.json @@ -0,0 +1,11 @@ +{ + "services": [ + { + "serviceName": "DynamoDBv2", + "type": "minor", + "changeLogMessages": [ + "Add native support for LINQ expression trees in the IDynamoDBContext API for ScanAsync() and QueryAsync()" + ] + } + ] +} \ No newline at end of file From 3865c1a87d21b3adc9baf0653879e245233772c7 Mon Sep 17 00:00:00 2001 From: irina-herciu Date: Thu, 12 Jun 2025 12:28:45 +0300 Subject: [PATCH 07/13] small refactoring --- .../DynamoDBv2/Custom/DataModel/Configs.cs | 4 +- .../Custom/DataModel/ContextExpression.cs | 62 ++++++++++--------- .../Custom/DataModel/ContextInternal.cs | 14 +++-- .../Custom/DataModel/QueryConfig.cs | 4 +- .../_async/IDynamoDBContext.Async.cs | 6 +- .../DataModel/_bcl/IDynamoDBContext.Sync.cs | 6 +- .../IntegrationTests/DataModelTests.cs | 14 ++--- .../UnitTests/Custom/ContextInternalTests.cs | 2 +- 8 files changed, 62 insertions(+), 50 deletions(-) diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs index 46d737f8cfe5..8f053e8b5617 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/Configs.cs @@ -275,7 +275,7 @@ public class DynamoDBOperationConfig /// /// Note: Conditions must be against non-key properties. /// - public ContextExpression ExpressionFilter { get; set; } + public ContextExpression Expression { get; set; } /// /// Default constructor @@ -292,7 +292,7 @@ public DynamoDBOperationConfig() internal void ValidateFilter() { - if (QueryFilter is { Count: > 0 } && ExpressionFilter is { Filter: not null } ) + if (QueryFilter is { Count: > 0 } && Expression is { Filter: not null } ) { throw new InvalidOperationException("Cannot specify both QueryFilter and ExpressionFilter in the same operation configuration. Please use one or the other."); } diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs index 271645489ba8..99be4a08703f 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs @@ -8,20 +8,23 @@ namespace Amazon.DynamoDBv2.DataModel { /// /// Represents a context expression for DynamoDB operations in the object-persistence programming model. + /// Used to encapsulate filter expressions for query and scan operations. /// public class ContextExpression { /// - /// Represents a filter expression that can be used to filter results in DynamoDB operations. + /// Gets the filter expression used to filter results in DynamoDB operations. + /// This expression is typically constructed from a LINQ expression tree. /// public Expression Filter { get; private set; } /// /// Sets the filter expression for DynamoDB operations. + /// Converts the provided LINQ expression into an internal expression tree for use in DynamoDB queries or scans. /// - /// - /// - /// + /// The type of the object being filtered. + /// A LINQ expression representing the filter condition. + /// Thrown if is null. public void SetFilter(Expression> filterExpression) { if (filterExpression == null) @@ -33,48 +36,49 @@ public void SetFilter(Expression> filterExpression) } /// - /// Extensions for LINQ operations in DynamoDB. + /// Provides extension methods for use in LINQ-to-DynamoDB expression trees. + /// These methods are intended for query translation and should not be called directly at runtime. /// public static class LinqDdbExtensions { /// - /// Checks if a value is between two other values, inclusive. - /// - /// This method is only used inside expression trees; it should never be called at runtime. + /// Indicates that the value should be compared to see if it falls inclusively between the specified lower and upper bounds. + /// Intended for use in LINQ expressions to generate DynamoDB BETWEEN conditions. + /// This method is only used inside expression trees and should not be called at runtime. /// - /// - /// - /// - /// - /// + /// The type of the value being compared. + /// The value to test. + /// The inclusive lower bound. + /// The inclusive upper bound. + /// True if the value is between the bounds; otherwise, false. public static bool Between(this T value, T lower, T upper) => throw null!; /// - /// Checks if a value is not between two other values, inclusive. - /// - /// This method is only used inside expression trees; it should never be called at runtime. + /// Indicates that the attribute exists in the DynamoDB item. + /// Intended for use in LINQ expressions to generate DynamoDB attribute_exists conditions. + /// This method is only used inside expression trees and should not be called at runtime. /// - /// - /// + /// The object representing the attribute to check. + /// True if the attribute exists; otherwise, false. public static bool AttributeExists(this object _) => throw null!; /// - /// Checks if a value does not have a specific attribute. - /// - /// This method is only used inside expression trees; it should never be called at runtime. + /// Indicates that the attribute does not exist in the DynamoDB item. + /// Intended for use in LINQ expressions to generate DynamoDB attribute_not_exists conditions. + /// This method is only used inside expression trees and should not be called at runtime. /// - /// - /// + /// The object representing the attribute to check. + /// True if the attribute does not exist; otherwise, false. public static bool AttributeNotExists(this object _) => throw null!; /// - /// Checks if a value has a specific attribute type. - /// - /// This method is only used inside expression trees; it should never be called at runtime. + /// Indicates that the attribute is of the specified DynamoDB type. + /// Intended for use in LINQ expressions to generate DynamoDB attribute_type conditions. + /// This method is only used inside expression trees and should not be called at runtime. /// - /// - /// - /// + /// The object representing the attribute to check. + /// The DynamoDB attribute type to compare against. + /// True if the attribute is of the specified type; otherwise, false. public static bool AttributeType(this object _, DynamoDBAttributeType dynamoDbType) => throw null!; } diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs index 27b6b5ca1901..912bb8619e31 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs @@ -1405,6 +1405,10 @@ public ContextSearch(Search search, DynamoDBFlatConfig flatConfig) DocumentModel.Expression expression = null; if (filterExpression is not { Filter: null }) { + if (flatConfig.QueryFilter != null && flatConfig.QueryFilter.Count != 0) + { + throw new InvalidOperationException("QueryFilter is not supported with filter expression. Use either QueryFilter or filter expression, but not both."); + } expression = ComposeExpression(filterExpression.Filter, storageConfig, flatConfig); } @@ -1454,11 +1458,11 @@ public ContextSearch(Search search, DynamoDBFlatConfig flatConfig) DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); - //todo - add support for expression + ContextSearch query; - if (operationConfig is { ExpressionFilter: { Filter: not null } }) + if (operationConfig is { Expression: { Filter: not null } }) { - query = ConvertQueryByValueWithExpression(hashKeyValue, op, values, operationConfig.ExpressionFilter.Filter, operationConfig, storageConfig); + query = ConvertQueryByValueWithExpression(hashKeyValue, op, values, operationConfig.Expression.Filter, operationConfig, storageConfig); } else { @@ -1498,10 +1502,10 @@ internal ContextSearch storageConfig ??= StorageConfigCache.GetConfig(flatConfig); ContextSearch query; - if (operationConfig is { ExpressionFilter: { Filter: not null } }) + if (operationConfig is { Expression: { Filter: not null } }) { query = ConvertQueryByValueWithExpression(hashKeyValue, QueryOperator.Equal, null, - operationConfig.ExpressionFilter.Filter, operationConfig, storageConfig); + operationConfig.Expression.Filter, operationConfig, storageConfig); } else { diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/QueryConfig.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/QueryConfig.cs index 4999f76fe819..82b334002186 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/QueryConfig.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/QueryConfig.cs @@ -70,7 +70,7 @@ public class QueryConfig : BaseOperationConfig /// /// Note: Expression filters must be against non-key properties. /// - public ContextExpression ExpressionFilter { get; set; } + public ContextExpression Expression { get; set; } /// /// Property that directs to use consistent reads. @@ -99,7 +99,7 @@ internal override DynamoDBOperationConfig ToDynamoDBOperationConfig() config.IndexName = IndexName; config.ConditionalOperator = ConditionalOperator; config.QueryFilter = QueryFilter; - config.ExpressionFilter = ExpressionFilter; + config.Expression = Expression; config.ConsistentRead = ConsistentRead; config.RetrieveDateTimeInUtc = RetrieveDateTimeInUtc; diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/_async/IDynamoDBContext.Async.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/_async/IDynamoDBContext.Async.cs index 674b7f358a66..4e0f65902f01 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/_async/IDynamoDBContext.Async.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/_async/IDynamoDBContext.Async.cs @@ -506,7 +506,8 @@ partial interface IDynamoDBContext /// /// Type of object. /// - /// A LINQ expression used to filter the results. The expression is translated to a DynamoDB filter expression and applied server-side. + /// A representing a LINQ expression tree used to filter the results. + /// The expression is translated to a DynamoDB filter expression and applied server-side. /// /// AsyncSearch which can be used to retrieve DynamoDB data. IAsyncSearch ScanAsync<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(ContextExpression filterExpression); @@ -543,7 +544,8 @@ partial interface IDynamoDBContext /// /// Type of object. /// - /// A LINQ expression used to filter the results. The expression is translated to a DynamoDB filter expression and applied server-side. + /// A representing a LINQ expression tree used to filter the results. + /// The expression is translated to a DynamoDB filter expression and applied server-side. /// /// Config object that can be used to override properties on the table's context for this request. /// AsyncSearch which can be used to retrieve DynamoDB data. diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/_bcl/IDynamoDBContext.Sync.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/_bcl/IDynamoDBContext.Sync.cs index 2885c272f088..bd62db611f49 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/_bcl/IDynamoDBContext.Sync.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/_bcl/IDynamoDBContext.Sync.cs @@ -419,7 +419,8 @@ partial interface IDynamoDBContext /// /// Type of object. /// - /// A LINQ expression used to filter the results. The expression is translated to a DynamoDB filter expression and applied server-side. + /// A representing a LINQ expression tree used to filter the results. + /// The expression is translated to a DynamoDB filter expression and applied server-side. /// /// /// A lazy-loaded collection of results of type that match the filter expression. @@ -458,7 +459,8 @@ partial interface IDynamoDBContext /// /// Type of object. /// - /// A LINQ expression used to filter the results. The expression is translated to a DynamoDB filter expression and applied server-side. + /// A representing a LINQ expression tree used to filter the results. + /// The expression is translated to a DynamoDB filter expression and applied server-side. /// /// Config object that can be used to override properties on the table's context for this request. /// Lazy-loaded collection of results. diff --git a/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs b/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs index 024e9d8b9428..dfec6a3a1b05 100644 --- a/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs +++ b/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs @@ -897,7 +897,7 @@ public void TestContext_Query_WithExpressionFilter() "Alice", new QueryConfig { - ExpressionFilter = contextExpression + Expression = contextExpression }).ToList(); Assert.AreEqual(1, employees.Count); @@ -907,7 +907,7 @@ public void TestContext_Query_WithExpressionFilter() "Charlie", new QueryConfig { - ExpressionFilter = contextExpression + Expression = contextExpression }).ToList(); Assert.AreEqual(1, employees.Count); @@ -917,7 +917,7 @@ public void TestContext_Query_WithExpressionFilter() "Bob", new QueryConfig { - ExpressionFilter = contextExpression + Expression = contextExpression }).ToList(); Assert.AreEqual(0, employees.Count); @@ -978,7 +978,7 @@ public void TestContext_Query_QueryFilter_vs_ExpressionFilter() contextExpression.SetFilter(e => e.ManagerName == "Eva"); var resultExpressionFilter = Context.Query("Diane", new QueryConfig { - ExpressionFilter = contextExpression + Expression = contextExpression }).ToList(); // Assert both results are equivalent @@ -1006,7 +1006,7 @@ public void TestContext_Query_QueryFilter_vs_ExpressionFilter() }).ToList(); var resultBarbara = Context.Query("Diane", new QueryConfig { - ExpressionFilter = contextExpressionBarbara + Expression = contextExpressionBarbara }).ToList(); Assert.AreEqual(resultActive.Count, resultBarbara.Count, "Result counts should match between QueryFilter and ExpressionFilter."); @@ -1032,7 +1032,7 @@ public void TestContext_Query_QueryFilter_vs_ExpressionFilter() var resultOrExpressionFilter = Context.Query("Diane", new QueryConfig { - ExpressionFilter = contextExpressionOr + Expression = contextExpressionOr }).ToList(); // Assert both results are equivalent @@ -1047,7 +1047,7 @@ public void TestContext_Query_QueryFilter_vs_ExpressionFilter() var resultIndex = Context.Query("Big River", new QueryConfig { IndexName = "GlobalIndex", - ExpressionFilter = contextExpression + Expression = contextExpression }).ToList(); Assert.AreEqual(2, resultIndex.Count); Assert.IsTrue(resultIndex.All(e => e.ManagerName == "Eva")); diff --git a/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextInternalTests.cs b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextInternalTests.cs index f5d25ef536de..43eaa59a0720 100644 --- a/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextInternalTests.cs +++ b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextInternalTests.cs @@ -185,7 +185,7 @@ public void ConvertQueryByValue_WithHashKeyAndExpressionFilter() var operationConfig = new DynamoDBOperationConfig { - ExpressionFilter = filterExpr + Expression = filterExpr }; // Act From 4f93f280cb44b9175a7a0881d90759ce1bfb8304 Mon Sep 17 00:00:00 2001 From: irina-herciu Date: Thu, 12 Jun 2025 17:03:52 +0300 Subject: [PATCH 08/13] clenup --- .../Custom/DataModel/ContextExpression.cs | 5 +++-- .../Custom/DataModel/ContextInternal.cs | 18 +++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs index 99be4a08703f..3c1cb71c04ae 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Linq.Expressions; +using ThirdParty.RuntimeBackports; using Expression = System.Linq.Expressions.Expression; namespace Amazon.DynamoDBv2.DataModel @@ -25,7 +26,7 @@ public class ContextExpression /// The type of the object being filtered. /// A LINQ expression representing the filter condition. /// Thrown if is null. - public void SetFilter(Expression> filterExpression) + public void SetFilter<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(Expression> filterExpression) { if (filterExpression == null) { @@ -51,7 +52,7 @@ public static class LinqDdbExtensions /// The inclusive lower bound. /// The inclusive upper bound. /// True if the value is between the bounds; otherwise, false. - public static bool Between(this T value, T lower, T upper) => throw null!; + public static bool Between<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(this T value, T lower, T upper) => throw null!; /// /// Indicates that the attribute exists in the DynamoDB item. diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs index 912bb8619e31..8f8171f92be6 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs @@ -1462,7 +1462,8 @@ public ContextSearch(Search search, DynamoDBFlatConfig flatConfig) ContextSearch query; if (operationConfig is { Expression: { Filter: not null } }) { - query = ConvertQueryByValueWithExpression(hashKeyValue, op, values, operationConfig.Expression.Filter, operationConfig, storageConfig); + query = ConvertQueryByValueWithExpression(hashKeyValue, op, values, operationConfig.Expression.Filter, + operationConfig, storageConfig); } else { @@ -1504,6 +1505,10 @@ internal ContextSearch ContextSearch query; if (operationConfig is { Expression: { Filter: not null } }) { + if(conditions!=null && conditions.Any()) + { + throw new InvalidOperationException("Query conditions are not supported with filter expression. Use either Query conditions or filter expression, but not both."); + } query = ConvertQueryByValueWithExpression(hashKeyValue, QueryOperator.Equal, null, operationConfig.Expression.Filter, operationConfig, storageConfig); } @@ -1540,17 +1545,13 @@ internal ContextSearch } }; - string rangeKeyPropertyName; - string indexName = flatConfig.IndexName; - if (string.IsNullOrEmpty(indexName)) - rangeKeyPropertyName = storageConfig.RangeKeyPropertyNames.FirstOrDefault(); - else - rangeKeyPropertyName = storageConfig.GetRangeKeyByIndex(indexName); + + var rangeKeyPropertyName = string.IsNullOrEmpty(indexName) ? + storageConfig.RangeKeyPropertyNames.FirstOrDefault() : storageConfig.GetRangeKeyByIndex(indexName); if (!string.IsNullOrEmpty(rangeKeyPropertyName) && values != null) { - //todo implement QueryOperator to expression mapping keyExpression.ExpressionStatement += ContextExpressionsUtils.GetRangeKeyConditionExpression($"#rangeKey", op); keyExpression.ExpressionAttributeNames.Add("#rangeKey", rangeKeyPropertyName); var valuesList = values?.ToList(); @@ -1589,7 +1590,6 @@ internal ContextSearch var expression = ComposeExpression(filterExpression, storageConfig, flatConfig); - //TODO string indexName = GetQueryIndexName(currentConfig, indexNames); queryConfig.FilterExpression = expression; if (string.IsNullOrEmpty(indexName)) From d7293c9e9743b0781e581d9c19bceb68257c0e27 Mon Sep 17 00:00:00 2001 From: irina-herciu Date: Thu, 19 Jun 2025 16:35:27 +0300 Subject: [PATCH 09/13] increase test coverage and address PR feedback --- .../Custom/DataModel/ContextExpression.cs | 87 ++++++++++---- .../Custom/DataModel/ContextInternal.cs | 5 +- .../IntegrationTests/DataModelTests.cs | 27 ++++- .../Custom/ContextExpressionsUtilsTests.cs | 109 ++++++++++++++++++ 4 files changed, 199 insertions(+), 29 deletions(-) diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs index 3c1cb71c04ae..29dbcbafdf2b 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs @@ -3,6 +3,9 @@ using System.Collections.Generic; using System.Linq.Expressions; using ThirdParty.RuntimeBackports; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Reflection; using Expression = System.Linq.Expressions.Expression; namespace Amazon.DynamoDBv2.DataModel @@ -34,14 +37,7 @@ public class ContextExpression } Filter = filterExpression.Body; } - } - /// - /// Provides extension methods for use in LINQ-to-DynamoDB expression trees. - /// These methods are intended for query translation and should not be called directly at runtime. - /// - public static class LinqDdbExtensions - { /// /// Indicates that the value should be compared to see if it falls inclusively between the specified lower and upper bounds. /// Intended for use in LINQ expressions to generate DynamoDB BETWEEN conditions. @@ -51,8 +47,8 @@ public static class LinqDdbExtensions /// The value to test. /// The inclusive lower bound. /// The inclusive upper bound. - /// True if the value is between the bounds; otherwise, false. - public static bool Between<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(this T value, T lower, T upper) => throw null!; + /// This method is intended to be used only within expression definitions (such as LINQ expression trees) and should not be called at runtime. + public static bool Between<[DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] T>(T value, T lower, T upper) => throw null!; /// /// Indicates that the attribute exists in the DynamoDB item. @@ -61,7 +57,7 @@ public static class LinqDdbExtensions /// /// The object representing the attribute to check. /// True if the attribute exists; otherwise, false. - public static bool AttributeExists(this object _) => throw null!; + public static bool AttributeExists(object _) => throw null!; /// /// Indicates that the attribute does not exist in the DynamoDB item. @@ -69,8 +65,8 @@ public static class LinqDdbExtensions /// This method is only used inside expression trees and should not be called at runtime. /// /// The object representing the attribute to check. - /// True if the attribute does not exist; otherwise, false. - public static bool AttributeNotExists(this object _) => throw null!; + /// This method is intended to be used only within expression definitions (such as LINQ expression trees) and should not be called at runtime. + public static bool AttributeNotExists(object _) => throw null!; /// /// Indicates that the attribute is of the specified DynamoDB type. @@ -79,8 +75,8 @@ public static class LinqDdbExtensions /// /// The object representing the attribute to check. /// The DynamoDB attribute type to compare against. - /// True if the attribute is of the specified type; otherwise, false. - public static bool AttributeType(this object _, DynamoDBAttributeType dynamoDbType) => throw null!; + /// This method is intended to be used only within expression definitions (such as LINQ expression trees) and should not be called at runtime. + public static bool AttributeType(object _, string dynamoDbType) => throw null!; } /// @@ -140,10 +136,45 @@ internal static ConstantExpression GetConstant(Expression expr) // If the expression is a UnaryExpression, check its Operand UnaryExpression unary => unary.Operand as ConstantExpression, NewExpression => throw new NotSupportedException($"Unsupported expression type {expr.Type}"), + MemberExpression member => GetConstantFromMember(member), _ => null }; } + private static ConstantExpression GetConstantFromMember(MemberExpression member) + { + var memberExpression= member.Expression; + var memberName= member.Member.Name; + if (memberExpression==null) + { + throw new InvalidOperationException("MemberExpression does not have an associated expression."); + } + var constant= GetConstant(memberExpression); + + var value= constant?.Value; + if (value != null) + { + // Use reflection to get the value of the member + var memberInfo = value.GetType().GetMember(memberName, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic).FirstOrDefault(); + if (memberInfo is FieldInfo field) + { + var fieldValue = field.GetValue(value); + return Expression.Constant(fieldValue, field.FieldType); + } + else if (memberInfo is PropertyInfo property) + { + var propertyValue = property.GetValue(value); + return Expression.Constant(propertyValue, property.PropertyType); + } + else + { + throw new InvalidOperationException($"Member '{memberName}' not found on type '{value.GetType()}'."); + } + } + + return constant ?? throw new InvalidOperationException($"Cannot extract constant from MemberExpression: {member}"); + } + internal static bool IsComparison(ExpressionType type) { return type is ExpressionType.Equal or ExpressionType.NotEqual or @@ -153,15 +184,27 @@ ExpressionType.GreaterThan or ExpressionType.GreaterThanOrEqual or internal static MemberExpression GetMember(Expression expr) { - if (expr is MemberExpression memberExpr) - return memberExpr; - - if (expr is UnaryExpression ue) - return GetMember(ue.Operand); + switch (expr) + { + case MemberExpression memberExpr: + return memberExpr; + case UnaryExpression ue: + return GetMember(ue.Operand); + // Handle indexer access (get_Item) for lists/arrays/dictionaries + case MethodCallExpression methodCall: + switch (methodCall.Method.Name) + { + case "get_Item": + return GetMember(methodCall.Object); + case "First": + case "FirstOrDefault": + if (methodCall.Arguments.Count > 0) + return GetMember(methodCall.Arguments[0]); + break; + } - // Handle indexer access (get_Item) for lists/arrays/dictionaries - if (expr is MethodCallExpression methodCall && methodCall.Method.Name == "get_Item") - return GetMember(methodCall.Object); + break; + } return null; } diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs index 8f8171f92be6..a0af9ca1ab3b 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs @@ -1791,8 +1791,9 @@ private ExpressionNode HandleAttributeTypeMethodCall(MethodCallExpression expr, if (expr.Arguments.Count == 2 && expr.Object == null) { - if (expr.Arguments[0] is MemberExpression memberObj && - expr.Arguments[1] is ConstantExpression typeExpr) + var memberObj = ContextExpressionsUtils.GetMember(expr.Arguments[0]); + var typeExpr = ContextExpressionsUtils.GetConstant(expr.Arguments[1]); + if (memberObj!=null && typeExpr!=null) { SetExpressionNodeAttributes(storageConfig, memberObj, typeExpr, node, flatConfig); } diff --git a/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs b/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs index dfec6a3a1b05..1cf6f3958263 100644 --- a/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs +++ b/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs @@ -689,7 +689,6 @@ public void TestContext_ScanWithExpression_NestedPaths() Assert.AreEqual("CloudSpotter", byDictionaryNested[0].Name); } - [TestMethod] [TestCategory("DynamoDBv2")] public void TestContext_Scan_WithExpressionFilter() @@ -736,8 +735,9 @@ public void TestContext_Scan_WithExpressionFilter() Context.Save(employee4); // Numeric equality + int age = 45; var exprAgeEq = new ContextExpression(); - exprAgeEq.SetFilter(e => e.Age == 45); + exprAgeEq.SetFilter(e => e.Age == age); var ageEqResult = Context.Scan(exprAgeEq).ToList(); Assert.AreEqual(2, ageEqResult.Count); @@ -810,7 +810,7 @@ public void TestContext_Scan_WithExpressionFilter() // Between var exprBetween = new ContextExpression(); - exprBetween.SetFilter(e => e.Age.Between(40, 50)); + exprBetween.SetFilter(e => ContextExpression.Between(e.Age, 40, 50)); var betweenResult = Context.Scan(exprBetween).ToList(); Assert.AreEqual(3, betweenResult.Count); Assert.IsTrue(betweenResult.All(e => e.Age >= 40 && e.Age <= 50)); @@ -822,15 +822,32 @@ public void TestContext_Scan_WithExpressionFilter() Assert.IsTrue(stringContainsResult.Any(e => e.Name == "Bob" || e.Name == "Rob" || e.Name == "Cob")); var exprNullCheck = new ContextExpression(); - exprNullCheck.SetFilter(e => e.MiddleName.AttributeExists()); + exprNullCheck.SetFilter(e => ContextExpression.AttributeExists(e.MiddleName)); var nullCheckResult = Context.Scan(exprNullCheck).ToList(); Assert.IsTrue(nullCheckResult.Count == 1); var exprNull = new ContextExpression(); - exprNull.SetFilter(e => e.MiddleName.AttributeNotExists()); + exprNull.SetFilter(e => ContextExpression.AttributeNotExists(e.MiddleName)); var nullResult = Context.Scan(exprNull).ToList(); Assert.IsTrue(nullResult.Count == 4); + //AttributeType scenario: filter for employees where MiddleName is a DynamoDB String + var empWithStringMiddleName = new Employee + { + Name = "TypeTest", + Age = 55, + CurrentStatus = Status.Inactive, + MiddleName = "StringType", + CompanyName = "test" + }; + Context.Save(empWithStringMiddleName); + + var attributeType = DynamoDBAttributeType.S.Value; + var exprAttributeType = new ContextExpression(); + exprAttributeType.SetFilter(e => ContextExpression.AttributeType(e.MiddleName, attributeType)); + var attributeTypeResult = Context.Scan(exprAttributeType).ToList(); + Assert.IsTrue(attributeTypeResult.Any(e => e.Name == "TypeTest")); + // --- Enum scenario --- // Scan for employees with CurrentStatus == Status.Active var exprActiveEnum = new ContextExpression(); diff --git a/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs index ec4b00c2f2fe..d4ae943a260a 100644 --- a/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs +++ b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs @@ -78,6 +78,59 @@ public void GetConstant_ReturnsNullForUnsupported() Assert.IsNull(ContextExpressionsUtils.GetConstant(paramExpr)); } + [TestMethod] + public void GetConstant_ReturnsConstantFromMember_Field() + { + var testObj = new TestClass { Field = 123 }; + Expression> expr = () => testObj.Field; + var memberExpr = (MemberExpression)expr.Body; + var result = ContextExpressionsUtils.GetConstant(memberExpr); + Assert.IsNotNull(result); + Assert.AreEqual(123, result.Value); + } + + [TestMethod] + public void GetConstant_ReturnsConstantFromMember_Property() + { + var testObj = new TestClass { Property = 456 }; + Expression> expr = () => testObj.Property; + var memberExpr = (MemberExpression)expr.Body; + var result = ContextExpressionsUtils.GetConstant(memberExpr); + Assert.IsNotNull(result); + Assert.AreEqual(456, result.Value); + } + + [TestMethod] + public void GetConstant_ThrowsForUnsupportedNewExpression() + { + Expression expr = Expression.New(typeof(TestClass)); + Assert.ThrowsException(() => ContextExpressionsUtils.GetConstant(expr)); + } + + [TestMethod] + public void GetConstant_ReturnsConstantFromNestedMember() + { + var inner = new TestClass { Field = 99, Property = 100 }; + var outer = new NestedTestClass { Inner = inner }; + Expression> expr = () => outer.Inner.Field; + var memberExpr = (MemberExpression)expr.Body; + var result = ContextExpressionsUtils.GetConstant(memberExpr); + Assert.IsNotNull(result); + Assert.AreEqual(99, result.Value); + } + + class NestedTestClass + { + public TestClass Inner { get; set; } + } + + class TestClass + { + public int Field; + public int Property { get; set; } + public List Array { get; set; } + } + [TestMethod] public void IsComparison_ReturnsTrueForComparisonTypes() { @@ -102,6 +155,62 @@ public void GetMember_ReturnsNullForNonMember() Assert.IsNull(ContextExpressionsUtils.GetMember(expr.Body)); } + [TestMethod] + public void GetMember_ReturnsMemberExpression_Direct() + { + Expression> expr = t => t.Field; + var memberExpr = expr.Body as MemberExpression; + var result = ContextExpressionsUtils.GetMember(expr.Body); + Assert.AreEqual(memberExpr, result); + } + + [TestMethod] + public void GetMember_ReturnsMemberExpression_FromUnary() + { + Expression> expr = t => (object)t.Property; + var result = ContextExpressionsUtils.GetMember(expr.Body); + Assert.IsNotNull(result); + Assert.AreEqual("Property", result.Member.Name); + } + + [TestMethod] + public void GetMember_ReturnsMemberExpression_FirstOrDefault() + { + Expression> expr = l => l.Array.FirstOrDefault() == true; + var methodCall = expr.Body as BinaryExpression; + var result = ContextExpressionsUtils.GetMember(methodCall.Left); + Assert.IsNotNull(result); + Assert.AreEqual("Array", result.Member.Name); + } + + [TestMethod] + public void GetMember_ReturnsMemberExpression_FromIndexer() + { + Expression> expr = l => l.Array[0] == true; + var methodCall = expr.Body as BinaryExpression; + var result = ContextExpressionsUtils.GetMember(methodCall.Left); + Assert.IsNotNull(result); + Assert.AreEqual("Array", result.Member.Name); + } + + [TestMethod] + public void GetMember_ReturnsMemberExpression_First() + { + Expression> expr = l => l.Array.First() == true; + var methodCall = expr.Body as BinaryExpression; + var result = ContextExpressionsUtils.GetMember(methodCall.Left); + Assert.IsNotNull(result); + Assert.AreEqual("Array", result.Member.Name); + } + + [TestMethod] + public void GetMember_ReturnsNull_ForNonMember() + { + Expression> expr = () => 1 + 2; + var result = ContextExpressionsUtils.GetMember(expr.Body); + Assert.IsNull(result); + } + [TestMethod] public void ExtractPathNodes_PropertyPath() { From 11c48e800e084ab652c62fbafbf05bd417cb7917 Mon Sep 17 00:00:00 2001 From: irina-herciu Date: Fri, 20 Jun 2025 10:24:59 +0300 Subject: [PATCH 10/13] add unit test for QueryFilter usage on Scan methods --- .../IntegrationTests/DataModelTests.cs | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs b/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs index 1cf6f3958263..406e8d5496ef 100644 --- a/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs +++ b/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs @@ -689,6 +689,58 @@ public void TestContext_ScanWithExpression_NestedPaths() Assert.AreEqual("CloudSpotter", byDictionaryNested[0].Name); } + [TestMethod] + [TestCategory("DynamoDBv2")] + public void TestContext_ScanConfigFilter() + { + TableCache.Clear(); + CleanupTables(); + TableCache.Clear(); + + var employee = new Employee() + { + Name = "Bob", + Age = 45, + CurrentStatus = Status.Active, + CompanyName = "test", + }; + + var employee3 = new Employee + { + Name = "Cob", + Age = 45, + CurrentStatus = Status.Inactive, + CompanyName = "test1", + }; + + + Context.Save(employee); + Context.Save(employee3); + + var ageEqResultScan = Context.Scan(new List(), new ScanConfig() + { + QueryFilter = new List() + { + new ScanCondition("Age", ScanOperator.GreaterThan,50) + }, + ConditionalOperator = ConditionalOperatorValues.And + }).ToList(); + Assert.AreEqual(0, ageEqResultScan.Count); + + var ageAndCompanyResultScan = Context.Scan(new List() + { + new ScanCondition("Age", ScanOperator.Equal,45) + }, new ScanConfig() + { + QueryFilter = new List() + { + new ScanCondition("CompanyName", ScanOperator.Equal, "Test") + }, + ConditionalOperator = ConditionalOperatorValues.And + }).ToList(); + Assert.AreEqual(1, ageEqResultScan.Count); + } + [TestMethod] [TestCategory("DynamoDBv2")] public void TestContext_Scan_WithExpressionFilter() From 0955526a2eb7a3fe27e533bcbfa0bb3585041556 Mon Sep 17 00:00:00 2001 From: irina-herciu Date: Fri, 20 Jun 2025 13:02:38 +0300 Subject: [PATCH 11/13] fix QueryFilter usage from scanConfig --- .../Custom/DataModel/ContextInternal.cs | 48 ++++++++++++------- .../IntegrationTests/DataModelTests.cs | 4 +- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs index a0af9ca1ab3b..a54417668ec4 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs @@ -1035,29 +1035,44 @@ private static bool TryGetValue(object instance, MemberInfo member, out object v private ScanFilter ComposeScanFilter(IEnumerable conditions, ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) { ScanFilter filter = new ScanFilter(); + + var conditionsToUse = new List(); if (conditions != null) { - foreach (var condition in conditions) + conditionsToUse.AddRange(conditions); + } + if (flatConfig.QueryFilter != null) + { + conditionsToUse.AddRange(flatConfig.QueryFilter); + } + + foreach (var condition in conditionsToUse) + { + PropertyStorage propertyStorage = + storageConfig.BaseTypeStorageConfig.GetPropertyStorage(condition.PropertyName); + List attributeValues = new List(); + foreach (var value in condition.Values) { - PropertyStorage propertyStorage = storageConfig.BaseTypeStorageConfig.GetPropertyStorage(condition.PropertyName); - List attributeValues = new List(); - foreach (var value in condition.Values) + var entry = ToDynamoDBEntry(propertyStorage, value, flatConfig, canReturnScalarInsteadOfList: true); + if (entry == null) + throw new InvalidOperationException( + string.Format(CultureInfo.InvariantCulture, + "Unable to convert value corresponding to property [{0}] to DynamoDB representation", + condition.PropertyName)); + + var attributeConversionConfig = + new DynamoDBEntry.AttributeConversionConfig(flatConfig.Conversion, + flatConfig.IsEmptyStringValueEnabled); + AttributeValue nativeValue = entry.ConvertToAttributeValue(attributeConversionConfig); + if (nativeValue != null) { - var entry = ToDynamoDBEntry(propertyStorage, value, flatConfig, canReturnScalarInsteadOfList: true); - if (entry == null) - throw new InvalidOperationException( - string.Format(CultureInfo.InvariantCulture, "Unable to convert value corresponding to property [{0}] to DynamoDB representation", condition.PropertyName)); - - var attributeConversionConfig = new DynamoDBEntry.AttributeConversionConfig(flatConfig.Conversion, flatConfig.IsEmptyStringValueEnabled); - AttributeValue nativeValue = entry.ConvertToAttributeValue(attributeConversionConfig); - if (nativeValue != null) - { - attributeValues.Add(nativeValue); - } + attributeValues.Add(nativeValue); } - filter.AddCondition(propertyStorage.AttributeName, condition.Operator, attributeValues); } + + filter.AddCondition(propertyStorage.AttributeName, condition.Operator, attributeValues); } + return filter; } @@ -1514,7 +1529,6 @@ internal ContextSearch } else { - List indexNames; QueryFilter filter = ComposeQueryFilter(flatConfig, hashKeyValue, conditions, storageConfig, out indexNames); query = ConvertQueryHelper(flatConfig, storageConfig, filter, indexNames); diff --git a/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs b/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs index 406e8d5496ef..95c71dfff7f0 100644 --- a/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs +++ b/sdk/test/Services/DynamoDBv2/IntegrationTests/DataModelTests.cs @@ -734,11 +734,11 @@ public void TestContext_ScanConfigFilter() { QueryFilter = new List() { - new ScanCondition("CompanyName", ScanOperator.Equal, "Test") + new ScanCondition("CompanyName", ScanOperator.Equal, "test") }, ConditionalOperator = ConditionalOperatorValues.And }).ToList(); - Assert.AreEqual(1, ageEqResultScan.Count); + Assert.AreEqual(1, ageAndCompanyResultScan.Count); } [TestMethod] From 94a65b5646acba213696515c79df40c5f6820493 Mon Sep 17 00:00:00 2001 From: irina-herciu Date: Fri, 20 Jun 2025 17:37:35 +0300 Subject: [PATCH 12/13] fix native aot for dictionary path --- .../Custom/DataModel/ContextExpression.cs | 104 ++++++++++-------- .../Custom/DataModel/ContextInternal.cs | 37 +++++-- 2 files changed, 87 insertions(+), 54 deletions(-) diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs index 29dbcbafdf2b..43f20da98f20 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs @@ -79,27 +79,7 @@ public class ContextExpression public static bool AttributeType(object _, string dynamoDbType) => throw null!; } - /// - /// Represents a node in a path expression for DynamoDB operations. - /// - internal class PathNode - { - public string Path { get; } - - public string FormattedPath { get; } - - public int IndexDepth { get; } - - public bool IsMap { get; } - - public PathNode(string path, int indexDepth, bool isMap, string formattedPath) - { - Path = path; - IndexDepth = indexDepth; - IsMap = isMap; - FormattedPath = formattedPath; - } - } + internal static class ContextExpressionsUtils { @@ -127,7 +107,7 @@ internal static bool IsMember(Expression expr) _ => false }; } - + internal static ConstantExpression GetConstant(Expression expr) { return expr switch @@ -141,40 +121,48 @@ internal static ConstantExpression GetConstant(Expression expr) }; } - private static ConstantExpression GetConstantFromMember(MemberExpression member) + private static ConstantExpression GetConstantFromMember( + MemberExpression member) { - var memberExpression= member.Expression; - var memberName= member.Member.Name; - if (memberExpression==null) + var memberExpression = member.Expression; + var memberName = member.Member.Name; + if (memberExpression == null) { throw new InvalidOperationException("MemberExpression does not have an associated expression."); } - var constant= GetConstant(memberExpression); + var constant = GetConstant(memberExpression); - var value= constant?.Value; + var value = constant?.Value; if (value != null) { - // Use reflection to get the value of the member - var memberInfo = value.GetType().GetMember(memberName, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic).FirstOrDefault(); - if (memberInfo is FieldInfo field) - { - var fieldValue = field.GetValue(value); - return Expression.Constant(fieldValue, field.FieldType); - } - else if (memberInfo is PropertyInfo property) - { - var propertyValue = property.GetValue(value); - return Expression.Constant(propertyValue, property.PropertyType); - } - else - { - throw new InvalidOperationException($"Member '{memberName}' not found on type '{value.GetType()}'."); - } + return ConstantFromMember(value, memberName); } return constant ?? throw new InvalidOperationException($"Cannot extract constant from MemberExpression: {member}"); } + private static ConstantExpression ConstantFromMember( + object value, string memberName) + { + var type = value.GetType(); + var memberInfo = Utils.GetMembersFromType(type).FirstOrDefault(); + + if (memberInfo is FieldInfo field) + { + var fieldValue = field.GetValue(value); + return Expression.Constant(fieldValue, field.FieldType); + } + else if (memberInfo is PropertyInfo property) + { + var propertyValue = property.GetValue(value); + return Expression.Constant(propertyValue, property.PropertyType); + } + else + { + throw new InvalidOperationException($"Member '{memberName}' not found on type '{value.GetType()}'."); + } + } + internal static bool IsComparison(ExpressionType type) { return type is ExpressionType.Equal or ExpressionType.NotEqual or @@ -282,4 +270,32 @@ internal static List ExtractPathNodes(Expression expr) return pathNodes; } } + + /// + /// Represents a node in a path expression for DynamoDB operations. + /// + internal class PathNode + { + public string Path { get; } + + public string FormattedPath { get; } + + public int IndexDepth { get; } + + public bool IsMap { get; } + + public PathNode(string path, int indexDepth, bool isMap, string formattedPath) + { + Path = path; + IndexDepth = indexDepth; + IsMap = isMap; + FormattedPath = formattedPath; + } + } + + internal class PropertyNode + { + [DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] + public Type PropertyType { get; set; } + } } \ No newline at end of file diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs index a54417668ec4..b477d1af1005 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs @@ -1473,7 +1473,7 @@ public ContextSearch(Search search, DynamoDBFlatConfig flatConfig) DynamoDBFlatConfig flatConfig = new DynamoDBFlatConfig(operationConfig, Config); ItemStorageConfig storageConfig = StorageConfigCache.GetConfig(flatConfig); - + ContextSearch query; if (operationConfig is { Expression: { Filter: not null } }) { @@ -2087,18 +2087,17 @@ private PropertyStorage ResolveNestedPropertyStorage(StorageConfig rootConfig, D depth += nextPathNode.IndexDepth; } - var nodePropertyType = propertyType; + var node = new PropertyNode() + { + PropertyType = propertyType + }; var currentDepth = 0; - while (currentDepth <= depth && nodePropertyType != null && Utils.ImplementsInterface(nodePropertyType, typeof(ICollection<>)) - && nodePropertyType != typeof(string)) + while (currentDepth <= depth && node.PropertyType != null && Utils.ImplementsInterface(node.PropertyType, typeof(ICollection<>)) + && node.PropertyType != typeof(string)) { - elementType = Utils.GetElementType(nodePropertyType); - if (elementType == null) - { - IsSupportedDictionaryType(nodePropertyType, out elementType); - } - nodePropertyType = elementType; + elementType = Utils.GetElementType(node.PropertyType) ?? GetDictionaryValueType(node.PropertyType); + node.PropertyType = elementType; currentDepth++; } elementType ??= propertyType; @@ -2110,6 +2109,24 @@ private PropertyStorage ResolveNestedPropertyStorage(StorageConfig rootConfig, D return propertyStorage; } + + [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2073", + Justification = "The user's type has been annotated with DynamicallyAccessedMemberTypes.All with the public API into the library. At this point the type will not be trimmed.")] + [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2063", + Justification = "The user's type has been annotated with DynamicallyAccessedMemberTypes.All with the public API into the library. At this point the type will not be trimmed.")] + [return: DynamicallyAccessedMembers(InternalConstants.DataModelModeledType)] + private static Type GetDictionaryValueType([DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.Interfaces)] Type type) + { + if (!(Utils.ImplementsInterface(type, typeof(IDictionary<,>)) && + Utils.ImplementsInterface(type, typeof(IDictionary)))) { } + else + { + var genericArguments = type.GetGenericArguments(); + if (genericArguments != null && genericArguments.Length == 2) + return genericArguments[1]; + } + return null; + } private PropertyStorage SetExpressionNameNode(ItemStorageConfig storageConfig, Expression memberObj, ExpressionNode node, DynamoDBFlatConfig flatConfig) { From bfe864209b6a3fa5a1fb34c808c664fa1fafba7e Mon Sep 17 00:00:00 2001 From: irina-herciu Date: Mon, 23 Jun 2025 12:57:20 +0300 Subject: [PATCH 13/13] fix native AOT worning on GetConstant --- .../Custom/DataModel/ContextExpression.cs | 164 ++++++++++-------- .../Custom/DataModel/ContextInternal.cs | 38 ++-- .../Custom/ContextExpressionsUtilsTests.cs | 96 +++++++++- 3 files changed, 206 insertions(+), 92 deletions(-) diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs index 43f20da98f20..6b0bf85108c2 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextExpression.cs @@ -79,7 +79,7 @@ public class ContextExpression public static bool AttributeType(object _, string dynamoDbType) => throw null!; } - + internal static class ContextExpressionsUtils { @@ -107,60 +107,56 @@ internal static bool IsMember(Expression expr) _ => false }; } - - internal static ConstantExpression GetConstant(Expression expr) + + internal static object GetConstant(Expression expr) { - return expr switch - { - ConstantExpression constant => constant, - // If the expression is a UnaryExpression, check its Operand - UnaryExpression unary => unary.Operand as ConstantExpression, - NewExpression => throw new NotSupportedException($"Unsupported expression type {expr.Type}"), - MemberExpression member => GetConstantFromMember(member), - _ => null - }; + return EvaluateExpression(expr); } - private static ConstantExpression GetConstantFromMember( - MemberExpression member) + private static object EvaluateExpression(Expression expr) { - var memberExpression = member.Expression; - var memberName = member.Member.Name; - if (memberExpression == null) + switch (expr) { - throw new InvalidOperationException("MemberExpression does not have an associated expression."); - } - var constant = GetConstant(memberExpression); + case ConstantExpression c: + return c.Value; - var value = constant?.Value; - if (value != null) - { - return ConstantFromMember(value, memberName); - } + case MemberExpression m: + var instance = m.Expression != null ? EvaluateExpression(m.Expression) : null; + if (m.Member is FieldInfo fi) + return fi.GetValue(instance); + if (m.Member is PropertyInfo pi) + return pi.GetValue(instance); + break; - return constant ?? throw new InvalidOperationException($"Cannot extract constant from MemberExpression: {member}"); - } + case MethodCallExpression call: + var method = call.Method; - private static ConstantExpression ConstantFromMember( - object value, string memberName) - { - var type = value.GetType(); - var memberInfo = Utils.GetMembersFromType(type).FirstOrDefault(); + if (method.Name == "get_Item") + { + var target = EvaluateExpression(call.Object); + var indexArgs = call.Arguments.Select(EvaluateExpression).ToArray(); + return method.Invoke(target, indexArgs); + } - if (memberInfo is FieldInfo field) - { - var fieldValue = field.GetValue(value); - return Expression.Constant(fieldValue, field.FieldType); - } - else if (memberInfo is PropertyInfo property) - { - var propertyValue = property.GetValue(value); - return Expression.Constant(propertyValue, property.PropertyType); - } - else - { - throw new InvalidOperationException($"Member '{memberName}' not found on type '{value.GetType()}'."); + if (method.Name == "Contains") + { + throw new NotSupportedException("The 'Contains' method is not supported for constant extraction in expression trees. Use supported property or indexer access instead."); + } + + var targetObj = call.Object != null ? EvaluateExpression(call.Object) : null; + var arguments = call.Arguments.Select(EvaluateExpression).ToArray(); + return method.Invoke(targetObj, arguments); + + case UnaryExpression u when u.NodeType == ExpressionType.Convert: + var operand = EvaluateExpression(u.Operand); + return Convert.ChangeType(operand, u.Type); + + case NewExpression n: + var args = n.Arguments.Select(EvaluateExpression).ToArray(); + return n.Constructor.Invoke(args); } + + throw new NotSupportedException($"Expression type '{expr.NodeType}' not supported."); } internal static bool IsComparison(ExpressionType type) @@ -208,7 +204,7 @@ internal static List ExtractPathNodes(Expression expr) switch (expr) { case MemberExpression memberExpr: - pathNodes.Insert(0, + pathNodes.Insert(0, new PathNode(memberExpr.Member.Name, indexDepth, false, $"#n{indexed}")); indexed = string.Empty; indexDepth = 0; @@ -220,36 +216,36 @@ internal static List ExtractPathNodes(Expression expr) indexed += "[0]"; break; case MethodCallExpression { Method: { Name: "get_Item" } } methodCall: - { - var arg = methodCall.Arguments[0]; - if (arg is ConstantExpression constArg) { - var indexValue = constArg.Value; - switch (indexValue) + var arg = methodCall.Arguments[0]; + if (arg is ConstantExpression constArg) { - case int intValue: - indexDepth++; - indexed += $"[{intValue}]"; - break; - case string stringValue: - pathNodes.Insert(0, new PathNode(stringValue, indexDepth, true, $"#n{indexed}")); - indexDepth = 0; - indexed = string.Empty; - break; - default: - throw new NotSupportedException( - $"Indexer argument must be an integer or string, got {indexValue.GetType().Name}."); + var indexValue = constArg.Value; + switch (indexValue) + { + case int intValue: + indexDepth++; + indexed += $"[{intValue}]"; + break; + case string stringValue: + pathNodes.Insert(0, new PathNode(stringValue, indexDepth, true, $"#n{indexed}")); + indexDepth = 0; + indexed = string.Empty; + break; + default: + throw new NotSupportedException( + $"Indexer argument must be an integer or string, got {indexValue.GetType().Name}."); + } + } + else + { + throw new NotSupportedException( + $"Method {methodCall.Method.Name} is not supported in property path."); } - } - else - { - throw new NotSupportedException( - $"Method {methodCall.Method.Name} is not supported in property path."); - } - expr = methodCall.Object; - break; - } + expr = methodCall.Object; + break; + } case MethodCallExpression methodCall: throw new NotSupportedException( $"Method {methodCall.Method.Name} is not supported in property path."); @@ -271,6 +267,30 @@ internal static List ExtractPathNodes(Expression expr) } } + //internal class ConstantExtractor : ExpressionVisitor + //{ + // public object ConstantValue { get; private set; } + + // protected override Expression VisitBinary(BinaryExpression node) + // { + // if (TryResolveValue(node.Right, out var value)) + // { + // ConstantValue = value; + // } + + // return base.VisitBinary(node); + // } + + // private bool TryResolveValue(Expression expr, out object value) + // { + // value = EvaluateExpression(expr); + // return true; + // } + + + + //} + /// /// Represents a node in a path expression for DynamoDB operations. /// diff --git a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs index b477d1af1005..d08002bd5e7d 100644 --- a/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs +++ b/sdk/src/Services/DynamoDBv2/Custom/DataModel/ContextInternal.cs @@ -1742,7 +1742,7 @@ private ExpressionNode BuildExpressionNode(Expression expr, ItemStorageConfig st private ExpressionNode HandleBinaryComparison(BinaryExpression expr, ItemStorageConfig storageConfig, DynamoDBFlatConfig flatConfig) { Expression member = null; - ConstantExpression constant = null; + object constant = null; if (ContextExpressionsUtils.IsMember(expr.Left)) { @@ -1927,8 +1927,9 @@ private ExpressionNode HandleBetweenMethodCall(MethodCallExpression expr, if (collectionExpr != null && constExprLeft != null && constExprRight != null) { var propertyStorage = SetExpressionNameNode(storageConfig, collectionExpr, node, flatConfig); - SetExpressionValueNode(constExprLeft, node, propertyStorage, flatConfig); - SetExpressionValueNode(constExprRight, node, propertyStorage, flatConfig); + + SetExpressionValueNode(ContextExpressionsUtils.GetConstant(constExprLeft), node, propertyStorage, flatConfig); + SetExpressionValueNode(ContextExpressionsUtils.GetConstant(constExprRight), node, propertyStorage, flatConfig); } } else @@ -1948,7 +1949,8 @@ private ExpressionNode HandleStartsWithMethodCall(MethodCallExpression expr, Ite }; if (expr.Object is MemberExpression memberObj && expr.Arguments[0] is ConstantExpression argConst) { - SetExpressionNodeAttributes(storageConfig, memberObj, argConst, node, flatConfig); + var constantValue=ContextExpressionsUtils.GetConstant(argConst); + SetExpressionNodeAttributes(storageConfig, memberObj, constantValue, node, flatConfig); } else { @@ -1967,7 +1969,7 @@ private ExpressionNode HandleContainsMethodCall(MethodCallExpression expr, }; if (expr.Object is MemberExpression memberObj && expr.Arguments[0] is ConstantExpression argConst) { - SetExpressionNodeAttributes(storageConfig, memberObj, argConst, node, flatConfig); + SetExpressionNodeAttributes(storageConfig, memberObj, ContextExpressionsUtils.GetConstant(argConst), node, flatConfig); } else if (expr.Arguments.Count == 2 && expr.Object == null) { @@ -1976,7 +1978,7 @@ private ExpressionNode HandleContainsMethodCall(MethodCallExpression expr, if (collectionExpr != null && constExpr != null) { - SetExpressionNodeAttributes(storageConfig, collectionExpr, constExpr, node, flatConfig); + SetExpressionNodeAttributes(storageConfig, collectionExpr, ContextExpressionsUtils.GetConstant(constExpr), node, flatConfig); } else { @@ -2011,10 +2013,20 @@ expr.Arguments[0] is ConstantExpression constant && } else if (expr.Arguments.Count == 2 && expr.Object == null) { - var memberObj = ContextExpressionsUtils.GetMember(expr.Arguments[0]) - ?? ContextExpressionsUtils.GetMember(expr.Arguments[1]); - var argConst = ContextExpressionsUtils.GetConstant(expr.Arguments[1]) - ?? ContextExpressionsUtils.GetConstant(expr.Arguments[0]); + Expression memberObj = null; + object argConst = null; + + if (ContextExpressionsUtils.IsMember(expr.Arguments[0])) + { + memberObj = expr.Arguments[0]; + argConst = ContextExpressionsUtils.GetConstant(expr.Arguments[1]); + } + else if (ContextExpressionsUtils.IsMember(expr.Arguments[1])) + { + memberObj = expr.Arguments[1]; + argConst = ContextExpressionsUtils.GetConstant(expr.Arguments[0]); + } + if (memberObj != null && argConst != null) { SetExpressionNodeAttributes(storageConfig, memberObj, argConst, node, flatConfig); @@ -2026,15 +2038,15 @@ expr.Arguments[0] is ConstantExpression constant && } private void SetExpressionNodeAttributes(ItemStorageConfig storageConfig, Expression memberObj, - ConstantExpression argConst, ExpressionNode node, DynamoDBFlatConfig flatConfig) + object argConst, ExpressionNode node, DynamoDBFlatConfig flatConfig) { var propertyStorage = SetExpressionNameNode(storageConfig, memberObj, node, flatConfig); SetExpressionValueNode(argConst, node, propertyStorage, flatConfig); } - private void SetExpressionValueNode(ConstantExpression argConst, ExpressionNode node, PropertyStorage propertyStorage, DynamoDBFlatConfig flatConfig) + private void SetExpressionValueNode(object argConst, ExpressionNode node, PropertyStorage propertyStorage, DynamoDBFlatConfig flatConfig) { - DynamoDBEntry entry = ToDynamoDBEntry(propertyStorage, argConst?.Value, flatConfig, canReturnScalarInsteadOfList: true); + DynamoDBEntry entry = ToDynamoDBEntry(propertyStorage, argConst, flatConfig, canReturnScalarInsteadOfList: true); var valuesNode = new ExpressionNode() { FormatedExpression = ExpressionFormatConstants.Value diff --git a/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs index d4ae943a260a..3923751eb146 100644 --- a/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs +++ b/sdk/test/Services/DynamoDBv2/UnitTests/Custom/ContextExpressionsUtilsTests.cs @@ -59,7 +59,7 @@ public void GetConstant_ReturnsConstantExpression() { var constExpr = Expression.Constant(42); var result = ContextExpressionsUtils.GetConstant(constExpr); - Assert.AreEqual(constExpr, result); + Assert.AreEqual(42, result); } [TestMethod] @@ -68,14 +68,14 @@ public void GetConstant_ReturnsConstantFromUnary() var constExpr = Expression.Constant(42); var unaryExpr = Expression.Convert(constExpr, typeof(object)); var result = ContextExpressionsUtils.GetConstant(unaryExpr); - Assert.AreEqual(constExpr, result); + Assert.AreEqual(42, result); } [TestMethod] public void GetConstant_ReturnsNullForUnsupported() { var paramExpr = Expression.Parameter(typeof(int), "x"); - Assert.IsNull(ContextExpressionsUtils.GetConstant(paramExpr)); + Assert.ThrowsException(() => ContextExpressionsUtils.GetConstant(paramExpr)); } [TestMethod] @@ -86,7 +86,7 @@ public void GetConstant_ReturnsConstantFromMember_Field() var memberExpr = (MemberExpression)expr.Body; var result = ContextExpressionsUtils.GetConstant(memberExpr); Assert.IsNotNull(result); - Assert.AreEqual(123, result.Value); + Assert.AreEqual(123, result); } [TestMethod] @@ -97,14 +97,14 @@ public void GetConstant_ReturnsConstantFromMember_Property() var memberExpr = (MemberExpression)expr.Body; var result = ContextExpressionsUtils.GetConstant(memberExpr); Assert.IsNotNull(result); - Assert.AreEqual(456, result.Value); + Assert.AreEqual(456, result); } [TestMethod] public void GetConstant_ThrowsForUnsupportedNewExpression() { Expression expr = Expression.New(typeof(TestClass)); - Assert.ThrowsException(() => ContextExpressionsUtils.GetConstant(expr)); + ContextExpressionsUtils.GetConstant(expr); } [TestMethod] @@ -116,7 +116,89 @@ public void GetConstant_ReturnsConstantFromNestedMember() var memberExpr = (MemberExpression)expr.Body; var result = ContextExpressionsUtils.GetConstant(memberExpr); Assert.IsNotNull(result); - Assert.AreEqual(99, result.Value); + Assert.AreEqual(99, result); + } + + [TestMethod] + public void GetConstant_ReturnsStaticField() + { + Expression expr = Expression.Field(null, typeof(Math).GetField(nameof(Math.PI))); + var result = ContextExpressionsUtils.GetConstant(expr); + Assert.AreEqual(Math.PI, result); + } + + [TestMethod] + public void GetConstant_ReturnsStaticProperty() + { + Expression expr = Expression.Property(null, typeof(DateTime).GetProperty(nameof(DateTime.Now))); + var result = ContextExpressionsUtils.GetConstant(expr); + Assert.IsInstanceOfType(result, typeof(DateTime)); + } + + [TestMethod] + public void GetConstant_ReturnsListIndexer() + { + var list = new List { 10, 20, 30 }; + Expression expr = Expression.Call( + Expression.Constant(list), + typeof(List).GetMethod("get_Item"), + Expression.Constant(1)); + var result = ContextExpressionsUtils.GetConstant(expr); + Assert.AreEqual(20, result); + } + + [TestMethod] + public void GetConstant_ReturnsDictionaryIndexer() + { + var dict = new Dictionary { { "a", 1 }, { "b", 2 } }; + Expression expr = Expression.Call( + Expression.Constant(dict), + typeof(Dictionary).GetMethod("get_Item"), + Expression.Constant("b")); + var result = ContextExpressionsUtils.GetConstant(expr); + Assert.AreEqual(2, result); + } + + [TestMethod] + public void GetConstant_ReturnsMethodCallWithArguments() + { + var str = "hello"; + Expression expr = Expression.Call( + Expression.Constant(str), + typeof(string).GetMethod("Substring", new[] { typeof(int), typeof(int) }), + Expression.Constant(1), + Expression.Constant(2)); + var result = ContextExpressionsUtils.GetConstant(expr); + Assert.AreEqual("el", result); + } + + [TestMethod] + public void GetConstant_ReturnsStaticMethodCall() + { + Expression expr = Expression.Call( + null, + typeof(string).GetMethod("IsNullOrEmpty", new[] { typeof(string) }), + Expression.Constant("")); + var result = ContextExpressionsUtils.GetConstant(expr); + Assert.AreEqual(true, result); + } + + [TestMethod] + public void GetConstant_ReturnsUnaryConvertToString() + { + var constExpr = Expression.Constant(123); + var unaryExpr = Expression.Convert(constExpr, typeof(string), typeof(Convert).GetMethod("ToString", new[] { typeof(int) })); + var result = ContextExpressionsUtils.GetConstant(unaryExpr); + Assert.AreEqual("123", result.ToString()); + } + + [TestMethod] + public void GetConstant_ReturnsNewExpressionWithArguments() + { + Expression expr = Expression.New(typeof(TimeSpan).GetConstructor(new[] { typeof(int), typeof(int), typeof(int) }), + Expression.Constant(1), Expression.Constant(2), Expression.Constant(3)); + var result = ContextExpressionsUtils.GetConstant(expr); + Assert.AreEqual(new TimeSpan(1, 2, 3), result); } class NestedTestClass