Create secondary indices based on table bean annotations (#3923) (#4004)

* Create secondary indices based on table bean annotations (#3923)

* detect and group indices present in table schema into LSIs and GSIs
* pass request with indices information appended further

* Remove specifying provisioned throughput for GSIs (#3923)

* If there's no information about the billing mode of the new table,
  then it'll be using the PAY_PER_REQUEST one. It means that all
  GSIs related to this table will be doing the same and there's
  no need to hard code any provisioned throughput like it was done

* Allow passing empty indices list to CreateTableOperation (#3923)

* CreateTableRequest cannot handle empty list of indices of any type. It
  throws exception when given such a list. At the same time, it nicely
  handles the cases when indices lists are null. Make sure then that
  when empty indices list is passed CreateTableOperation, then in the
  CreateTableRequest it's just reflected as null.

---------

Co-authored-by: Adrian Chlebosz <[email protected]>
Co-authored-by: Olivier L Applin <[email protected]>

Author: 3 people

.changes/next-release/bugfix-AmazonDynamoDBEnhanced-66db474.json

@@ -0,0 +1,6 @@
+{
+    "category": "Amazon DynamoDB Enhanced", 
+    "contributor": "breader124", 
+    "type": "bugfix", 
+    "description": "Thanks to this bugfix it'll be possible to create DynamoDB table containing\nsecondary indices when using no arugments `createTable` method from `DefaultDynamoDbTable`\nclass. Information about their presence might be expressed using annotations, but it was ignored\nand created tables didn't contain specified indices. Plase note that it is still not possible\nto specify projections for indices using annotations. By default, all fields will be projected."
+}

services-custom/dynamodb-enhanced/src/main/java/software/amazon/awssdk/enhanced/dynamodb/internal/client/DefaultDynamoDbTable.java

@@ -15,13 +15,20 @@
 
 package software.amazon.awssdk.enhanced.dynamodb.internal.client;
 
+import static java.util.Collections.emptyList;
 import static software.amazon.awssdk.enhanced.dynamodb.internal.EnhancedClientUtils.createKeyFromItem;
 
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
 import software.amazon.awssdk.annotations.SdkInternalApi;
 import software.amazon.awssdk.enhanced.dynamodb.DynamoDbEnhancedClientExtension;
 import software.amazon.awssdk.enhanced.dynamodb.DynamoDbTable;
+import software.amazon.awssdk.enhanced.dynamodb.IndexMetadata;
 import software.amazon.awssdk.enhanced.dynamodb.Key;
+import software.amazon.awssdk.enhanced.dynamodb.KeyAttributeMetadata;
 import software.amazon.awssdk.enhanced.dynamodb.TableMetadata;
 import software.amazon.awssdk.enhanced.dynamodb.TableSchema;
 import software.amazon.awssdk.enhanced.dynamodb.internal.operations.CreateTableOperation;
@@ -39,6 +46,8 @@
 import software.amazon.awssdk.enhanced.dynamodb.model.DeleteItemEnhancedRequest;
 import software.amazon.awssdk.enhanced.dynamodb.model.DeleteItemEnhancedResponse;
 import software.amazon.awssdk.enhanced.dynamodb.model.DescribeTableEnhancedResponse;
+import software.amazon.awssdk.enhanced.dynamodb.model.EnhancedGlobalSecondaryIndex;
+import software.amazon.awssdk.enhanced.dynamodb.model.EnhancedLocalSecondaryIndex;
 import software.amazon.awssdk.enhanced.dynamodb.model.GetItemEnhancedRequest;
 import software.amazon.awssdk.enhanced.dynamodb.model.PageIterable;
 import software.amazon.awssdk.enhanced.dynamodb.model.PutItemEnhancedRequest;
@@ -51,6 +60,7 @@
 import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
 import software.amazon.awssdk.services.dynamodb.model.DescribeTableRequest;
 import software.amazon.awssdk.services.dynamodb.model.DescribeTableResponse;
+import software.amazon.awssdk.services.dynamodb.model.ProjectionType;
 
 @SdkInternalApi
 public class DefaultDynamoDbTable<T> implements DynamoDbTable<T> {
@@ -115,7 +125,51 @@ public void createTable(Consumer<CreateTableEnhancedRequest.Builder> requestCons
 
     @Override
     public void createTable() {
-        createTable(CreateTableEnhancedRequest.builder().build());
+        Map<IndexType, List<IndexMetadata>> indexGroups = splitSecondaryIndicesToLocalAndGlobalOnes();
+        createTable(CreateTableEnhancedRequest.builder()
+                                              .localSecondaryIndices(extractLocalSecondaryIndices(indexGroups))
+                                              .globalSecondaryIndices(extractGlobalSecondaryIndices(indexGroups))
+                                              .build());
+    }
+
+    private Map<IndexType, List<IndexMetadata>> splitSecondaryIndicesToLocalAndGlobalOnes() {
+        String primaryPartitionKeyName = tableSchema.tableMetadata().primaryPartitionKey();
+        Collection<IndexMetadata> indices = tableSchema.tableMetadata().indices();
+        return indices.stream()
+                      .filter(index -> !TableMetadata.primaryIndexName().equals(index.name()))
+                      .collect(Collectors.groupingBy(metadata -> {
+                          String partitionKeyName = metadata.partitionKey().map(KeyAttributeMetadata::name).orElse(null);
+                          if (partitionKeyName == null || primaryPartitionKeyName.equals(partitionKeyName)) {
+                              return IndexType.LSI;
+                          }
+                          return IndexType.GSI;
+                      }));
+    }
+
+    private List<EnhancedLocalSecondaryIndex> extractLocalSecondaryIndices(Map<IndexType, List<IndexMetadata>> indicesGroups) {
+        return indicesGroups.getOrDefault(IndexType.LSI, emptyList()).stream()
+                            .map(this::mapIndexMetadataToEnhancedLocalSecondaryIndex)
+                            .collect(Collectors.toList());
+    }
+
+    private EnhancedLocalSecondaryIndex mapIndexMetadataToEnhancedLocalSecondaryIndex(IndexMetadata indexMetadata) {
+        return EnhancedLocalSecondaryIndex.builder()
+                                          .indexName(indexMetadata.name())
+                                          .projection(pb -> pb.projectionType(ProjectionType.ALL))
+                                          .build();
+    }
+
+    private List<EnhancedGlobalSecondaryIndex> extractGlobalSecondaryIndices(Map<IndexType, List<IndexMetadata>> indicesGroups) {
+        return indicesGroups.getOrDefault(IndexType.GSI, emptyList()).stream()
+                            .map(this::mapIndexMetadataToEnhancedGlobalSecondaryIndex)
+                            .collect(Collectors.toList());
+    }
+
+    private EnhancedGlobalSecondaryIndex mapIndexMetadataToEnhancedGlobalSecondaryIndex(IndexMetadata indexMetadata) {
+        return EnhancedGlobalSecondaryIndex.builder()
+                                           .indexName(indexMetadata.name())
+                                           .projection(pb -> pb.projectionType(ProjectionType.ALL))
+                                           .build();
     }
 
     @Override

services-custom/dynamodb-enhanced/src/main/java/software/amazon/awssdk/enhanced/dynamodb/internal/client/IndexType.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.enhanced.dynamodb.internal.client;
+
+import software.amazon.awssdk.annotations.SdkInternalApi;
+
+/**
+ * Enum collecting types of secondary indexes
+ */
+@SdkInternalApi
+public enum IndexType {
+    LSI,
+    GSI
+}

services-custom/dynamodb-enhanced/src/main/java/software/amazon/awssdk/enhanced/dynamodb/internal/operations/CreateTableOperation.java

@@ -74,7 +74,7 @@ public CreateTableRequest generateRequest(TableSchema<T> tableSchema,
         List<software.amazon.awssdk.services.dynamodb.model.GlobalSecondaryIndex> sdkGlobalSecondaryIndices = null;
         List<software.amazon.awssdk.services.dynamodb.model.LocalSecondaryIndex> sdkLocalSecondaryIndices = null;
 
-        if (this.request.globalSecondaryIndices() != null) {
+        if (this.request.globalSecondaryIndices() != null && !this.request.globalSecondaryIndices().isEmpty()) {
             sdkGlobalSecondaryIndices =
                 this.request.globalSecondaryIndices().stream().map(gsi -> {
                     String indexPartitionKey = tableSchema.tableMetadata().indexPartitionKey(gsi.indexName());
@@ -92,7 +92,7 @@ public CreateTableRequest generateRequest(TableSchema<T> tableSchema,
                 }).collect(Collectors.toList());
         }
 
-        if (this.request.localSecondaryIndices() != null) {
+        if (this.request.localSecondaryIndices() != null && !this.request.localSecondaryIndices().isEmpty()) {
             sdkLocalSecondaryIndices =
                 this.request.localSecondaryIndices().stream().map(lsi -> {
                     Optional<String> indexSortKey = tableSchema.tableMetadata().indexSortKey(lsi.indexName());

services-custom/dynamodb-enhanced/src/test/java/software/amazon/awssdk/enhanced/dynamodb/internal/client/DefaultDynamoDbTableTest.java

@@ -16,20 +16,30 @@
 package software.amazon.awssdk.enhanced.dynamodb.internal.client;
 
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Mockito.verify;
 import static software.amazon.awssdk.enhanced.dynamodb.internal.AttributeValues.stringValue;
 
+import java.util.Iterator;
+import java.util.List;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
 import org.mockito.Mock;
+import org.mockito.Mockito;
 import org.mockito.junit.MockitoJUnitRunner;
 import software.amazon.awssdk.enhanced.dynamodb.DynamoDbEnhancedClientExtension;
 import software.amazon.awssdk.enhanced.dynamodb.Key;
 import software.amazon.awssdk.enhanced.dynamodb.functionaltests.models.FakeItem;
 import software.amazon.awssdk.enhanced.dynamodb.functionaltests.models.FakeItemWithIndices;
 import software.amazon.awssdk.enhanced.dynamodb.functionaltests.models.FakeItemWithSort;
+import software.amazon.awssdk.enhanced.dynamodb.model.CreateTableEnhancedRequest;
+import software.amazon.awssdk.enhanced.dynamodb.model.EnhancedGlobalSecondaryIndex;
+import software.amazon.awssdk.enhanced.dynamodb.model.EnhancedLocalSecondaryIndex;
 import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
 
 @RunWith(MockitoJUnitRunner.class)
@@ -113,4 +123,50 @@ public void keyFrom_primaryIndex_partitionAndNullSort() {
         assertThat(key.partitionKeyValue(), is(stringValue(item.getId())));
         assertThat(key.sortKeyValue(), is(Optional.empty()));
     }
+
+    @Test
+    public void createTable_doesNotTreatPrimaryIndexAsAnyOfSecondaryIndexes() {
+        DefaultDynamoDbTable<FakeItem> dynamoDbMappedIndex =
+            Mockito.spy(new DefaultDynamoDbTable<>(mockDynamoDbClient,
+                                                   mockDynamoDbEnhancedClientExtension,
+                                                   FakeItem.getTableSchema(),
+                                                   "test_table"));
+
+        dynamoDbMappedIndex.createTable();
+
+        CreateTableEnhancedRequest request = captureCreateTableRequest(dynamoDbMappedIndex);
+
+        assertThat(request.localSecondaryIndices().size(), is(0));
+        assertThat(request.globalSecondaryIndices().size(), is(0));
+    }
+
+    @Test
+    public void createTable_groupsSecondaryIndexesExistingInTableSchema() {
+        DefaultDynamoDbTable<FakeItemWithIndices> dynamoDbMappedIndex =
+            Mockito.spy(new DefaultDynamoDbTable<>(mockDynamoDbClient,
+                                                   mockDynamoDbEnhancedClientExtension,
+                                                   FakeItemWithIndices.getTableSchema(),
+                                                   "test_table"));
+
+        dynamoDbMappedIndex.createTable();
+
+        CreateTableEnhancedRequest request = captureCreateTableRequest(dynamoDbMappedIndex);
+
+        assertThat(request.localSecondaryIndices().size(), is(1));
+        Iterator<EnhancedLocalSecondaryIndex> lsiIterator = request.localSecondaryIndices().iterator();
+        assertThat(lsiIterator.next().indexName(), is("lsi_1"));
+
+        assertThat(request.globalSecondaryIndices().size(), is(2));
+        List<String> globalIndicesNames = request.globalSecondaryIndices().stream()
+                                                 .map(EnhancedGlobalSecondaryIndex::indexName)
+                                                 .collect(Collectors.toList());
+        assertThat(globalIndicesNames, containsInAnyOrder("gsi_1", "gsi_2"));
+    }
+
+    private static <T> CreateTableEnhancedRequest captureCreateTableRequest(DefaultDynamoDbTable<T> index) {
+        ArgumentCaptor<CreateTableEnhancedRequest> createTableOperationCaptor =
+            ArgumentCaptor.forClass(CreateTableEnhancedRequest.class);
+        verify(index).createTable(createTableOperationCaptor.capture());
+        return createTableOperationCaptor.getValue();
+    }
 }