Add support for filtering objects as part of S3 Transfer Manager's DownloadDirectory API (#3042)

* Add support for filtering objects as part of S3 Transfer Manager's DownloadDirectory API

This allows users to specify a filter that will be used to evaluate
which objects should be downloaded from the target S3 directory.

You can use a filter, for example, to only download objects of a given
size, of a given file extension, of a given last-modified date, etc. See
`DownloadFilter` for some ready-made implementations.

By default, if no filter is specified, all objects will be downloaded.

Author: Bennett Lynch

.changes/next-release/feature-S3TransferManager-d85514d.json

@@ -0,0 +1,6 @@
+{
+    "category": "S3 Transfer Manager", 
+    "contributor": "", 
+    "type": "feature", 
+    "description": "Add support for filtering objects as part of S3 Transfer Manager's DownloadDirectory API"
+}

services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3TransferManagerDownloadDirectoryIntegrationTest.java

@@ -16,24 +16,24 @@
 package software.amazon.awssdk.transfer.s3;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static software.amazon.awssdk.testutils.FileUtils.toFileTreeString;
 import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
 import static software.amazon.awssdk.utils.IoUtils.closeQuietly;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.ComparisonFailure;
 import org.junit.Test;
 import software.amazon.awssdk.testutils.FileUtils;
 import software.amazon.awssdk.utils.Logger;
@@ -109,12 +109,15 @@ public static void teardown() {
      * <pre>
      *   {@code
      *      - destination
-     *           - 2021
-     *              - 1.txt
-     *              - 2.txt
-     *           - 2022
-     *               - 1.txt
-     *           - important.txt
+     *          - README.md
+     *          - CHANGELOG.md
+     *          - notes
+     *              - 2021
+     *                  - 1.txt
+     *                  - 2.txt
+     *              - 2022
+     *                  - 1.txt
+     *              - important.txt
      *   }
      * </pre>
      */
@@ -131,16 +134,13 @@ public void downloadDirectory() throws Exception {
      * The destination directory structure should be the following with prefix "notes"
      * <pre>
      *   {@code
-     *      - source
-     *          - README.md
-     *          - CHANGELOG.md
-     *          - notes
-     *              - 2021
-     *                  - 1.txt
-     *                  - 2.txt
-     *              - 2022
-     *                  - 1.txt
-     *              - important.txt
+     *      - destination
+     *          - 2021
+     *              - 1.txt
+     *              - 2.txt
+     *          - 2022
+     *              - 1.txt
+     *          - important.txt
      *   }
      * </pre>
      */
@@ -178,24 +178,64 @@ public void downloadDirectory_withPrefixAndDelimiter() throws Exception {
         assertTwoDirectoriesHaveSameStructure(sourceDirectory.resolve("notes").resolve("2021"), destinationDirectory);
     }
 
-    private static void assertTwoDirectoriesHaveSameStructure(Path path, Path otherPath) {
+    /**
+     * The destination directory structure should only contain file names starting with "2":
+     * <pre>
+     *   {@code
+     *      - destination
+     *          - notes
+     *              - 2021
+     *                  - 2.txt
+     *   }
+     * </pre>
+     */
+    @Test
+    public void downloadDirectory_withFilter() throws Exception {
+        DirectoryDownload downloadDirectory = tm.downloadDirectory(u -> u
+            .destinationDirectory(destinationDirectory)
+            .bucket(TEST_BUCKET)
+            .filter(ctx -> ctx.destination().getFileName().toString().startsWith("2")));
+        CompletedDirectoryDownload completedDirectoryDownload = downloadDirectory.completionFuture().get(5, TimeUnit.SECONDS);
+        assertThat(completedDirectoryDownload.failedTransfers()).isEmpty();
+
+        Path expectedDirectory = Files.createTempDirectory("expectedDirectory");
         try {
-            Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
-                @Override
-                public FileVisitResult visitFile(Path file,
-                                                 BasicFileAttributes attrs)
-                    throws IOException {
-                    FileVisitResult result = super.visitFile(file, attrs);
-
-                    Path relativePath = path.relativize(file);
-                    Path otherFile = otherPath.resolve(relativePath);
-                    log.debug(() -> String.format("Comparing %s with %s", file, otherFile));
-                    assertThat(file).hasSameBinaryContentAs(otherFile);
-                    return result;
+            FileUtils.copyDirectory(sourceDirectory, expectedDirectory);
+            Files.delete(expectedDirectory.resolve("README.md"));
+            Files.delete(expectedDirectory.resolve("CHANGELOG.md"));
+            Files.delete(expectedDirectory.resolve("notes/2022/1.txt"));
+            Files.delete(expectedDirectory.resolve("notes/2022"));
+            Files.delete(expectedDirectory.resolve("notes/important.txt"));
+            Files.delete(expectedDirectory.resolve("notes/2021/1.txt"));
+            
+            assertTwoDirectoriesHaveSameStructure(expectedDirectory, destinationDirectory);
+        } finally {
+            FileUtils.cleanUpTestDirectory(expectedDirectory);
+        }
+    }
+
+    private static void assertTwoDirectoriesHaveSameStructure(Path a, Path b) {
+        assertLeftHasRight(a, b);
+        assertLeftHasRight(b, a);
+    }
+
+    private static void assertLeftHasRight(Path left, Path right) {
+        try (Stream<Path> paths = Files.walk(left)) {
+            paths.forEach(leftPath -> {
+                Path leftRelative = left.relativize(leftPath);
+                Path rightPath = right.resolve(leftRelative);
+                log.debug(() -> String.format("Comparing %s with %s", leftPath, rightPath));
+                try {
+                    assertThat(rightPath).exists();
+                } catch (AssertionError e) {
+                    throw new ComparisonFailure(e.getMessage(), toFileTreeString(left), toFileTreeString(right));
+                }
+                if (Files.isRegularFile(leftPath)) {
+                    assertThat(leftPath).hasSameBinaryContentAs(rightPath);
                 }
             });
         } catch (IOException e) {
-            throw new UncheckedIOException(String.format("Failed to compare %s with %s", path, otherPath), e);
+            throw new UncheckedIOException(String.format("Failed to compare %s with %s", left, right), e);
         }
     }
 

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadDirectoryRequest.java

@@ -40,12 +40,14 @@ public final class DownloadDirectoryRequest
     private final String bucket;
     private final String prefix;
     private final String delimiter;
+    private final DownloadFilter filter;
 
     public DownloadDirectoryRequest(DefaultBuilder builder) {
         this.destinationDirectory = Validate.paramNotNull(builder.destinationDirectory, "destinationDirectory");
         this.bucket = Validate.paramNotNull(builder.bucket, "bucket");
         this.prefix = builder.prefix;
         this.delimiter = builder.delimiter;
+        this.filter = builder.filter;
     }
 
     /**
@@ -84,6 +86,14 @@ public Optional<String> delimiter() {
         return Optional.ofNullable(delimiter);
     }
 
+    /**
+     * @return the optional filter, or {@link DownloadFilter#allObjects()} if no filter was provided
+     * @see Builder#filter(DownloadFilter)
+     */
+    public DownloadFilter filter() {
+        return filter == null ? DownloadFilter.allObjects() : filter;
+    }
+
     public static Builder builder() {
         return new DefaultBuilder();
     }
@@ -117,7 +127,10 @@ public boolean equals(Object o) {
         if (!Objects.equals(prefix, that.prefix)) {
             return false;
         }
-        return Objects.equals(delimiter, that.delimiter);
+        if (!Objects.equals(delimiter, that.delimiter)) {
+            return false;
+        }
+        return Objects.equals(filter, that.filter);
     }
 
     @Override
@@ -126,6 +139,7 @@ public int hashCode() {
         result = 31 * result + (bucket != null ? bucket.hashCode() : 0);
         result = 31 * result + (prefix != null ? prefix.hashCode() : 0);
         result = 31 * result + (delimiter != null ? delimiter.hashCode() : 0);
+        result = 31 * result + (filter != null ? filter.hashCode() : 0);
         return result;
     }
 
@@ -136,6 +150,7 @@ public String toString() {
                        .add("bucket", bucket)
                        .add("prefix", prefix)
                        .add("delimiter", delimiter)
+                       .add("filter", filter)
                        .build();
     }
 
@@ -236,8 +251,20 @@ public interface Builder extends CopyableBuilder<Builder, DownloadDirectoryReque
          */
         Builder delimiter(String delimiter);
 
-        @Override
-        DownloadDirectoryRequest build();
+        /**
+         * Specify a filter that will be used to evaluate which objects should be downloaded from the target directory.
+         * <p>
+         * You can use a filter, for example, to only download objects of a given size, of a given file extension, of a given
+         * last-modified date, etc. See {@link DownloadFilter} for some ready-made implementations. Multiple {@link
+         * DownloadFilter}s can be composed together via the {@code and} and {@code or} methods.
+         * <p>
+         * By default, if no filter is specified, all objects will be downloaded.
+         *
+         * @param filter the filter
+         * @return This builder for method chaining.
+         * @see DownloadFilter
+         */
+        Builder filter(DownloadFilter filter);
     }
 
     private static final class DefaultBuilder implements Builder {
@@ -246,6 +273,7 @@ private static final class DefaultBuilder implements Builder {
         private String bucket;
         private String prefix;
         private String delimiter;
+        private DownloadFilter filter;
 
         private DefaultBuilder() {
         }
@@ -254,6 +282,7 @@ private DefaultBuilder(DownloadDirectoryRequest request) {
             this.destinationDirectory = request.destinationDirectory;
             this.bucket = request.bucket;
             this.prefix = request.prefix;
+            this.filter = request.filter;
         }
 
         @Override
@@ -312,6 +341,20 @@ public String getDelimiter() {
             return delimiter;
         }
 
+        @Override
+        public Builder filter(DownloadFilter filter) {
+            this.filter = filter;
+            return this;
+        }
+
+        public void setFilter(DownloadFilter filter) {
+            filter(filter);
+        }
+
+        public DownloadFilter getFilter() {
+            return filter;
+        }
+
         @Override
         public DownloadDirectoryRequest build() {
             return new DownloadDirectoryRequest(this);

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadFileContext.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3;
+
+import java.nio.file.Path;
+import software.amazon.awssdk.annotations.SdkPreviewApi;
+import software.amazon.awssdk.annotations.SdkPublicApi;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+/**
+ * Context object for determining which objects should be downloaded as part of a {@link DownloadDirectoryRequest}.
+ *
+ * @see DownloadFilter
+ */
+@SdkPublicApi
+@SdkPreviewApi
+public interface DownloadFileContext {
+    /**
+     * @return A description of the remote S3 object
+     */
+    S3Object source();
+
+    /**
+     * @return A path representing the local file destination
+     */
+    Path destination();
+}

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/DownloadFilter.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.transfer.s3;
+
+import java.nio.file.Path;
+import java.util.function.Predicate;
+import software.amazon.awssdk.annotations.SdkPreviewApi;
+import software.amazon.awssdk.annotations.SdkPublicApi;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+/**
+ * {@link DownloadFilter} allows you to filter out which objects should be downloaded as part of a {@link
+ * DownloadDirectoryRequest}. You could use it, for example, to only download objects of a given size, of a given file extension,
+ * of a given last-modified date, etc. Multiple {@link DownloadFilter}s can be composed together via {@link #and(Predicate)} and
+ * {@link #or(Predicate)} methods.
+ */
+@SdkPublicApi
+@SdkPreviewApi
+public interface DownloadFilter extends Predicate<DownloadFileContext> {
+
+    /**
+     * Evaluate whether the remote {@link S3Object} should be downloaded to the destination {@link Path}.
+     *
+     * @param context A context object containing a description of the remote {@link S3Object} and a {@link Path} representing the
+     *                local file destination
+     * @return true if the object should be downloaded, false if the object should not be downloaded
+     */
+    @Override
+    boolean test(DownloadFileContext context);
+
+    /**
+     * A {@link DownloadFilter} that downloads all objects. This is the default behavior if no filter is provided.
+     */
+    @SdkPreviewApi
+    static DownloadFilter allObjects() {
+        return ctx -> true;
+    }
+}