diff --git a/.changes/next-release/bugfix-AmazonS3-6cefd9c.json b/.changes/next-release/bugfix-AmazonS3-6cefd9c.json new file mode 100644 index 000000000000..16c4151fcf33 --- /dev/null +++ b/.changes/next-release/bugfix-AmazonS3-6cefd9c.json @@ -0,0 +1,6 @@ +{ + "type": "bugfix", + "category": "Amazon S3", + "contributor": "", + "description": "Fixed an issue that could cause checksum mismatch errors when performing parallel uploads with the async S3 client and the SHA1 or SHA256 checksum algorithms selected." +} diff --git a/core/checksums/pom.xml b/core/checksums/pom.xml index 19839659ef6e..999a68809cec 100644 --- a/core/checksums/pom.xml +++ b/core/checksums/pom.xml @@ -63,6 +63,11 @@ junit-jupiter test + + org.assertj + assertj-core + test + diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java index 08e05b3ae6b9..e11f6fd2cc8c 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java @@ -21,9 +21,8 @@ import software.amazon.awssdk.checksums.internal.Crc32Checksum; import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum; import software.amazon.awssdk.checksums.internal.CrcChecksumProvider; -import software.amazon.awssdk.checksums.internal.Md5Checksum; -import software.amazon.awssdk.checksums.internal.Sha1Checksum; -import software.amazon.awssdk.checksums.internal.Sha256Checksum; +import software.amazon.awssdk.checksums.internal.DigestAlgorithm; +import software.amazon.awssdk.checksums.internal.DigestAlgorithmChecksum; import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm; /** @@ -43,11 +42,11 @@ static SdkChecksum forAlgorithm(ChecksumAlgorithm algorithm) { case "CRC32": return new Crc32Checksum(); case "SHA1": - return new Sha1Checksum(); + return new DigestAlgorithmChecksum(DigestAlgorithm.SHA1); case "SHA256": - return new Sha256Checksum(); + return new DigestAlgorithmChecksum(DigestAlgorithm.SHA256); case "MD5": - return new Md5Checksum(); + return new DigestAlgorithmChecksum(DigestAlgorithm.MD5); case "CRC64NVME": return new Crc64NvmeChecksum(); default: diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithm.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithm.java index cd5e63a070d3..56d7b7517f35 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithm.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithm.java @@ -17,22 +17,31 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.Deque; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.function.Supplier; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.annotations.SdkTestInternalApi; +import software.amazon.awssdk.utils.SdkAutoCloseable; @SdkInternalApi public enum DigestAlgorithm { - SHA1("SHA-1"), + MD5("MD5"), SHA256("SHA-256") ; + private static final Supplier CLOSED_DIGEST = () -> { + throw new IllegalStateException("This message digest is closed."); + }; + + private static final int MAX_CACHED_DIGESTS = 10_000; private final String algorithmName; - private final DigestThreadLocal digestReference; + private final Deque digestCache = new LinkedBlockingDeque<>(MAX_CACHED_DIGESTS); // LIFO DigestAlgorithm(String algorithmName) { this.algorithmName = algorithmName; - digestReference = new DigestThreadLocal(algorithmName); } public String getAlgorithmName() { @@ -40,28 +49,83 @@ public String getAlgorithmName() { } /** - * Returns the thread local reference for the {@link MessageDigest} algorithm + * Returns a {@link CloseableMessageDigest} to use for this algorithm. */ - public MessageDigest getDigest() { - MessageDigest digest = digestReference.get(); - digest.reset(); - return digest; + public CloseableMessageDigest getDigest() { + MessageDigest digest = digestCache.pollFirst(); + if (digest != null) { + digest.reset(); + return new CloseableMessageDigest(digest); + } + return new CloseableMessageDigest(newDigest()); } - private static class DigestThreadLocal extends ThreadLocal { - private final String algorithmName; + private MessageDigest newDigest() { + try { + return MessageDigest.getInstance(algorithmName); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("Unable to fetch message digest instance for Algorithm " + + algorithmName + ": " + e.getMessage(), e); + } + } + + @SdkTestInternalApi + static void clearCaches() { + for (DigestAlgorithm value : values()) { + value.digestCache.clear(); + } + } + + public final class CloseableMessageDigest implements SdkAutoCloseable, Cloneable { + + private Supplier digest; + private byte[] messageDigest; + + private CloseableMessageDigest(MessageDigest digest) { + this.digest = () -> digest; + } + + /** + * Retrieve the message digest instance. + */ + public MessageDigest messageDigest() { + return digest.get(); + } + + /** + * Retrieve the message digest bytes. This will close the message digest when invoked. This is because the underlying + * message digest is reset on read, and we'd rather fail future interactions with the digest than act on the wrong data. + */ + public byte[] digest() { + if (messageDigest != null) { + return messageDigest; + } + messageDigest = messageDigest().digest(); + close(); + return messageDigest; + } + + /** + * Release this message digest back to the cache. Once released, you must not use the digest anymore. + */ + @Override + public void close() { + if (digest == CLOSED_DIGEST) { + return; + } + + // Drop this digest is the cache is full. + digestCache.offerFirst(digest.get()); - DigestThreadLocal(String algorithmName) { - this.algorithmName = algorithmName; + digest = CLOSED_DIGEST; } @Override - protected MessageDigest initialValue() { + public CloseableMessageDigest clone() { try { - return MessageDigest.getInstance(algorithmName); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException("Unable to fetch message digest instance for Algorithm " - + algorithmName + ": " + e.getMessage(), e); + return new CloseableMessageDigest((MessageDigest) digest.get().clone()); + } catch (CloneNotSupportedException e) { + throw new IllegalStateException("Clone was not supported by this digest type.", e); } } } diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Md5Checksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithmChecksum.java similarity index 54% rename from core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Md5Checksum.java rename to core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithmChecksum.java index 6b1a5d7cc67b..3886b4cb5e4b 100644 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Md5Checksum.java +++ b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithmChecksum.java @@ -15,32 +15,39 @@ package software.amazon.awssdk.checksums.internal; -import java.security.MessageDigest; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.checksums.SdkChecksum; +import software.amazon.awssdk.checksums.internal.DigestAlgorithm.CloseableMessageDigest; /** - * Implementation of {@link SdkChecksum} to calculate an MD5 checksum. + * An implementation of {@link SdkChecksum} that uses a {@link DigestAlgorithm}. */ @SdkInternalApi -public class Md5Checksum implements SdkChecksum { +public class DigestAlgorithmChecksum implements SdkChecksum { - private MessageDigest digest; + private final DigestAlgorithm algorithm; - private MessageDigest digestLastMarked; + private CloseableMessageDigest digest; - public Md5Checksum() { - this.digest = getDigest(); + private CloseableMessageDigest digestLastMarked; + + public DigestAlgorithmChecksum(DigestAlgorithm algorithm) { + this.algorithm = algorithm; + this.digest = newDigest(); + } + + private CloseableMessageDigest newDigest() { + return algorithm.getDigest(); } @Override public void update(int b) { - digest.update((byte) b); + digest.messageDigest().update((byte) b); } @Override public void update(byte[] b, int off, int len) { - digest.update(b, off, len); + digest.messageDigest().update(b, off, len); } @Override @@ -50,15 +57,12 @@ public long getValue() { @Override public void reset() { - digest = (digestLastMarked == null) - // This is necessary so that should there be a reset without a - // preceding mark, the MD5 would still be computed correctly. - ? getDigest() - : cloneFrom(digestLastMarked); - } - - private MessageDigest getDigest() { - return DigestAlgorithm.MD5.getDigest(); + digest.close(); + if (digestLastMarked == null) { + digest = newDigest(); + } else { + digest = digestLastMarked; + } } @Override @@ -68,14 +72,6 @@ public byte[] getChecksumBytes() { @Override public void mark(int readLimit) { - digestLastMarked = cloneFrom(digest); - } - - private MessageDigest cloneFrom(MessageDigest from) { - try { - return (MessageDigest) from.clone(); - } catch (CloneNotSupportedException e) { // should never occur - throw new IllegalStateException("unexpected", e); - } + digestLastMarked = digest.clone(); } } diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Sha1Checksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Sha1Checksum.java deleted file mode 100644 index d377a669f2c1..000000000000 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Sha1Checksum.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.checksums.internal; - -import java.security.MessageDigest; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.checksums.SdkChecksum; - -/** - * Implementation of {@link SdkChecksum} to calculate an Sha-1 checksum. - */ -@SdkInternalApi -public class Sha1Checksum implements SdkChecksum { - - private MessageDigest digest; - - private MessageDigest digestLastMarked; - - public Sha1Checksum() { - this.digest = getDigest(); - } - - @Override - public void update(int b) { - digest.update((byte) b); - } - - @Override - public void update(byte[] b, int off, int len) { - digest.update(b, off, len); - } - - @Override - public long getValue() { - throw new UnsupportedOperationException("Use getChecksumBytes() instead."); - } - - @Override - public void reset() { - digest = (digestLastMarked == null) - // This is necessary so that should there be a reset without a - // preceding mark, the Sha-1 would still be computed correctly. - ? getDigest() - : cloneFrom(digestLastMarked); - } - - private MessageDigest getDigest() { - return DigestAlgorithm.SHA1.getDigest(); - } - - @Override - public byte[] getChecksumBytes() { - return digest.digest(); - } - - @Override - public void mark(int readLimit) { - digestLastMarked = cloneFrom(digest); - } - - private MessageDigest cloneFrom(MessageDigest from) { - try { - return (MessageDigest) from.clone(); - } catch (CloneNotSupportedException e) { // should never occur - throw new IllegalStateException("unexpected", e); - } - } -} diff --git a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Sha256Checksum.java b/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Sha256Checksum.java deleted file mode 100644 index 472375d74dfd..000000000000 --- a/core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Sha256Checksum.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package software.amazon.awssdk.checksums.internal; - -import java.security.MessageDigest; -import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.checksums.SdkChecksum; - -/** - * Implementation of {@link SdkChecksum} to calculate an Sha-256 Checksum. - */ -@SdkInternalApi -public class Sha256Checksum implements SdkChecksum { - - private MessageDigest digest; - - private MessageDigest digestLastMarked; - - public Sha256Checksum() { - this.digest = getDigest(); - } - - @Override - public void update(int b) { - digest.update((byte) b); - } - - @Override - public void update(byte[] b, int off, int len) { - digest.update(b, off, len); - } - - @Override - public long getValue() { - throw new UnsupportedOperationException("Use getChecksumBytes() instead."); - } - - @Override - public void reset() { - digest = (digestLastMarked == null) - // This is necessary so that should there be a reset without a - // preceding mark, the Sha-256 would still be computed correctly. - ? getDigest() - : cloneFrom(digestLastMarked); - } - - private MessageDigest getDigest() { - return DigestAlgorithm.SHA256.getDigest(); - } - - @Override - public byte[] getChecksumBytes() { - return digest.digest(); - } - - @Override - public void mark(int readLimit) { - digestLastMarked = cloneFrom(digest); - } - - private MessageDigest cloneFrom(MessageDigest from) { - try { - return (MessageDigest) from.clone(); - } catch (CloneNotSupportedException e) { // should never occur - throw new IllegalStateException("unexpected", e); - } - } -} diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/SdkChecksumTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/SdkChecksumTest.java new file mode 100644 index 000000000000..d06fde2c9327 --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/SdkChecksumTest.java @@ -0,0 +1,58 @@ +package software.amazon.awssdk.checksums; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class SdkChecksumTest { + + @ParameterizedTest + @ValueSource(strings = { + "CRC32C", + "CRC32", + "SHA1", + "SHA256", + "MD5", + "CRC64NVME" + }) + void checksumCachesAreParallelSafe(String algorithm) { + int numChecksums = 100; + List checksums = new ArrayList<>(numChecksums); + List checksumBytes = new ArrayList<>(numChecksums); + + for (int i = 0; i < numChecksums; i++) { + SdkChecksum checksum = SdkChecksum.forAlgorithm(() -> algorithm); + byte[] bytes = getRandomBytes(); + checksum.update(bytes); + checksumBytes.add(checksum.getChecksumBytes()); + + checksum = SdkChecksum.forAlgorithm(() -> algorithm); + checksum.reset(); + checksum.update(bytes); + checksums.add(checksum); + } + + SdkChecksum checksumToUpdate = checksums.get(0); + checksumToUpdate.update(getRandomBytes()); + byte[] newChecksumBytes = checksumToUpdate.getChecksumBytes(); + checksumBytes.set(0, newChecksumBytes); + + for (int j = 1; j < numChecksums; j++) { + SdkChecksum checksumToTest = checksums.get(j); + byte[] expected = checksumBytes.get(j); + byte[] actual = checksumToTest.getChecksumBytes(); + assertArrayEquals(expected, actual); + } + } + + private static byte[] getRandomBytes() { + byte[] randomBytes = new byte[1024]; + Random random = new Random(); + random.nextBytes(randomBytes); + return randomBytes; + } +} \ No newline at end of file diff --git a/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DigestAlgorithmTest.java b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DigestAlgorithmTest.java new file mode 100644 index 000000000000..98978fa9f6a9 --- /dev/null +++ b/core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DigestAlgorithmTest.java @@ -0,0 +1,172 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.checksums.internal; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.checksums.internal.DigestAlgorithm.CloseableMessageDigest; +import software.amazon.awssdk.utils.BinaryUtils; + +class DigestAlgorithmTest { + @BeforeEach + void clearCache() { + DigestAlgorithm.clearCaches(); + } + + @Test + void getAlgorithmName_returnsCorrectValue() { + assertThat(DigestAlgorithm.SHA1.getAlgorithmName()).isEqualTo("SHA-1"); + assertThat(DigestAlgorithm.MD5.getAlgorithmName()).isEqualTo("MD5"); + assertThat(DigestAlgorithm.SHA256.getAlgorithmName()).isEqualTo("SHA-256"); + } + + @Test + void getDigest_returnsMessageDigest() { + CloseableMessageDigest digest = DigestAlgorithm.SHA1.getDigest(); + assertThat(digest).isNotNull(); + assertThat(digest.messageDigest()).isNotNull(); + } + + @Test + void digestAlgorithms_useCorrectImplementation() { + String input = "Hello, World!"; + byte[] data = input.getBytes(StandardCharsets.UTF_8); + + // Test SHA1 + CloseableMessageDigest sha1Digest = DigestAlgorithm.SHA1.getDigest(); + sha1Digest.messageDigest().update(data); + byte[] sha1Hash = sha1Digest.digest(); + assertThat(sha1Hash).isNotNull(); + assertThat(BinaryUtils.toBase64(sha1Hash)).isEqualTo("CgqfKmdylCVXq1NV12r0Qvj2XgE="); + + // Test MD5 + CloseableMessageDigest md5Digest = DigestAlgorithm.MD5.getDigest(); + md5Digest.messageDigest().update(data); + byte[] md5Hash = md5Digest.digest(); + assertThat(md5Hash).isNotNull(); + assertThat(BinaryUtils.toBase64(md5Hash)).isEqualTo("ZajifYh5KDgxtmS9i38K1A=="); + + // Test SHA256 + CloseableMessageDigest sha256Digest = DigestAlgorithm.SHA256.getDigest(); + sha256Digest.messageDigest().update(data); + byte[] sha256Hash = sha256Digest.digest(); + assertThat(sha256Hash).isNotNull(); + assertThat(BinaryUtils.toBase64(sha256Hash)).isEqualTo("3/1gIbsr1bCvZ2KQgJ7DpTGR3YHH9wpLKGiKNiGCmG8="); + } + + @Test + void closedDigests_areClearedAndReused() { + CloseableMessageDigest digest1 = DigestAlgorithm.SHA1.getDigest(); + MessageDigest messageDigest1 = digest1.messageDigest(); + messageDigest1.update((byte) 'a'); + byte[] aDigest = digest1.digest(); + digest1.close(); + + CloseableMessageDigest digest2 = DigestAlgorithm.SHA1.getDigest(); + assertThat(digest2.messageDigest()).isSameAs(messageDigest1); + assertThat(digest2.digest()).isNotEqualTo(aDigest); + digest2.close(); + } + + @Test + void digestClone_clonesDigestContent() { + CloseableMessageDigest original = DigestAlgorithm.SHA1.getDigest(); + original.messageDigest().update((byte) 'a'); + + CloseableMessageDigest cloned = original.clone(); + + assertThat(cloned).isNotNull(); + assertThat(cloned.messageDigest()).isNotSameAs(original.messageDigest()); + assertThat(original.digest()).isEqualTo(cloned.digest()); + } + + @Test + void digestClones_behaveIndependently() { + CloseableMessageDigest original = DigestAlgorithm.SHA1.getDigest(); + CloseableMessageDigest cloned = original.clone(); + + assertThat(cloned).isNotNull(); + assertThat(cloned.messageDigest()).isNotSameAs(original.messageDigest()); + + // Test that both digests work independently + original.messageDigest().update((byte) 'a'); + + assertThat(original.digest()).isNotEqualTo(cloned.digest()); + } + + @Test + void readingDigest_closesDigest() { + CloseableMessageDigest digest = DigestAlgorithm.SHA1.getDigest(); + + digest.digest(); + + assertThatThrownBy(() -> digest.messageDigest()) + .isInstanceOf(IllegalStateException.class); + } + + @Test + void closeDigest_canBeDoneMultipleTimes() { + CloseableMessageDigest digest = DigestAlgorithm.SHA1.getDigest(); + digest.close(); + digest.close(); + digest.close(); + } + + @Test + void closedDigests_failMethodCalls() { + CloseableMessageDigest digest = DigestAlgorithm.SHA1.getDigest(); + digest.close(); + + assertThatThrownBy(() -> digest.messageDigest()) + .isInstanceOf(IllegalStateException.class); + assertThatThrownBy(() -> digest.clone()) + .isInstanceOf(IllegalStateException.class); + } + + @Test + void digestsCanBeRetrievedMultipleTimes() { + CloseableMessageDigest digest = DigestAlgorithm.SHA1.getDigest(); + String input = "Test Data"; + byte[] data = input.getBytes(StandardCharsets.UTF_8); + + digest.messageDigest().update(data); + byte[] firstHash = digest.digest(); + byte[] secondHash = digest.digest(); + + assertThat(firstHash).isEqualTo(secondHash); + } + + @Test + void testCacheLimit() { + // Test that we can release more than the size of the cache back to the cache. + List digests = new ArrayList<>(); + for (int i = 0; i < 10_001; i++) { + CloseableMessageDigest digest = DigestAlgorithm.SHA1.getDigest(); + digest.close(); + } + + // Still able to get new digests + CloseableMessageDigest digest = DigestAlgorithm.SHA1.getDigest(); + assertThat(digest).isNotNull(); + } +} \ No newline at end of file diff --git a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/SignerUtils.java b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/SignerUtils.java index ca737b3ddc7e..4821c8a91085 100644 --- a/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/SignerUtils.java +++ b/core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/SignerUtils.java @@ -22,7 +22,6 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -30,7 +29,7 @@ import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; import software.amazon.awssdk.annotations.SdkInternalApi; -import software.amazon.awssdk.checksums.internal.DigestAlgorithm; +import software.amazon.awssdk.checksums.SdkChecksum; import software.amazon.awssdk.http.ContentStreamProvider; import software.amazon.awssdk.http.Header; import software.amazon.awssdk.http.SdkHttpRequest; @@ -217,10 +216,6 @@ public static long moveContentLength(SdkHttpRequest.Builder request, InputStream return Long.parseLong(decodedContentLength.get()); } - private static MessageDigest getMessageDigestInstance() { - return DigestAlgorithm.SHA256.getDigest(); - } - public static InputStream getBinaryRequestPayloadStream(ContentStreamProvider streamProvider) { try { if (streamProvider == null) { @@ -234,14 +229,14 @@ public static InputStream getBinaryRequestPayloadStream(ContentStreamProvider st public static byte[] hash(InputStream input) { try { - MessageDigest md = getMessageDigestInstance(); + SdkChecksum md = sha256Checksum(); byte[] buf = new byte[4096]; int read = 0; while (read >= 0) { read = input.read(buf); md.update(buf, 0, read); } - return md.digest(); + return md.getChecksumBytes(); } catch (Exception e) { throw new RuntimeException("Unable to compute hash while signing request: ", e); } @@ -249,9 +244,9 @@ public static byte[] hash(InputStream input) { public static byte[] hash(ByteBuffer input) { try { - MessageDigest md = getMessageDigestInstance(); + SdkChecksum md = sha256Checksum(); md.update(input); - return md.digest(); + return md.getChecksumBytes(); } catch (Exception e) { throw new RuntimeException("Unable to compute hash while signing request: ", e); } @@ -259,9 +254,9 @@ public static byte[] hash(ByteBuffer input) { public static byte[] hash(byte[] data) { try { - MessageDigest md = getMessageDigestInstance(); + SdkChecksum md = sha256Checksum(); md.update(data); - return md.digest(); + return md.getChecksumBytes(); } catch (Exception e) { throw new RuntimeException("Unable to compute hash while signing request: ", e); } @@ -297,4 +292,8 @@ public static String getContentHash(SdkHttpRequest.Builder requestBuilder) { () -> new IllegalArgumentException("Content hash must be present in the '" + X_AMZ_CONTENT_SHA256 + "' header!") ); } + + private static SdkChecksum sha256Checksum() { + return SdkChecksum.forAlgorithm(() -> "SHA256"); + } } diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumInputStreamTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumInputStreamTest.java index 7e8795db90dd..de66fda66c3b 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumInputStreamTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumInputStreamTest.java @@ -26,7 +26,8 @@ import software.amazon.awssdk.checksums.internal.Crc32Checksum; import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum; import software.amazon.awssdk.checksums.SdkChecksum; -import software.amazon.awssdk.checksums.internal.Sha256Checksum; +import software.amazon.awssdk.checksums.internal.DigestAlgorithm; +import software.amazon.awssdk.checksums.internal.DigestAlgorithmChecksum; import software.amazon.awssdk.utils.BinaryUtils; class ChecksumInputStreamTest { @@ -37,7 +38,7 @@ void read_computesCorrectSha256() { String expectedDigest = "004c6bbd87e7fe70109b3bc23c8b1ab8f18a8bede0ed38c9233f6cdfd4f7b5d6"; ByteArrayInputStream backingStream = new ByteArrayInputStream(testString.getBytes(StandardCharsets.UTF_8)); - SdkChecksum checksum = new Sha256Checksum(); + SdkChecksum checksum = new DigestAlgorithmChecksum(DigestAlgorithm.SHA256); ChecksumInputStream inputStream = new ChecksumInputStream(backingStream, Collections.singleton(checksum)); readAll(inputStream); @@ -54,7 +55,7 @@ void read_withMultipleChecksums_shouldComputeCorrectChecksums() { String expectedCrc64Digest = "7c05fe704e3e02bc"; ByteArrayInputStream backingStream = new ByteArrayInputStream(testString.getBytes(StandardCharsets.UTF_8)); - SdkChecksum sha256Checksum = new Sha256Checksum(); + SdkChecksum sha256Checksum = new DigestAlgorithmChecksum(DigestAlgorithm.SHA256); SdkChecksum crc32Checksum = new Crc32Checksum(); SdkChecksum crc64NvmeChecksum = new Crc64NvmeChecksum(); ChecksumInputStream inputStream = new ChecksumInputStream(backingStream, Arrays.asList(sha256Checksum, crc32Checksum, crc64NvmeChecksum)); diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTckTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTckTest.java index c6a4be5fcecb..9dfbb3cc0666 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTckTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTckTest.java @@ -19,7 +19,8 @@ import java.util.Collections; import org.reactivestreams.Subscriber; import org.reactivestreams.tck.TestEnvironment; -import software.amazon.awssdk.checksums.internal.Sha256Checksum; +import software.amazon.awssdk.checksums.internal.DigestAlgorithm; +import software.amazon.awssdk.checksums.internal.DigestAlgorithmChecksum; /** * TCK verifiation test for {@link ChecksumSubscriber}. @@ -32,7 +33,7 @@ public ChecksumSubscriberTckTest() { @Override public Subscriber createSubscriber() { - return new ChecksumSubscriber(Collections.singleton(new Sha256Checksum())); + return new ChecksumSubscriber(Collections.singleton(new DigestAlgorithmChecksum(DigestAlgorithm.SHA256))); } @Override diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTest.java index 712e8a752698..8960f57b64ac 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/io/ChecksumSubscriberTest.java @@ -33,7 +33,8 @@ import software.amazon.awssdk.checksums.internal.Crc32Checksum; import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum; import software.amazon.awssdk.checksums.SdkChecksum; -import software.amazon.awssdk.checksums.internal.Sha256Checksum; +import software.amazon.awssdk.checksums.internal.DigestAlgorithm; +import software.amazon.awssdk.checksums.internal.DigestAlgorithmChecksum; import software.amazon.awssdk.utils.BinaryUtils; public class ChecksumSubscriberTest { @@ -43,7 +44,7 @@ public void checksum_computesCorrectSha256() { String testString = "AWS SDK for Java"; String expectedDigest = "004c6bbd87e7fe70109b3bc23c8b1ab8f18a8bede0ed38c9233f6cdfd4f7b5d6"; - SdkChecksum checksum = new Sha256Checksum(); + SdkChecksum checksum = new DigestAlgorithmChecksum(DigestAlgorithm.SHA256); ChecksumSubscriber subscriber = new ChecksumSubscriber(Collections.singleton(checksum)); Flowable publisher = Flowable.just(ByteBuffer.wrap(testString.getBytes(StandardCharsets.UTF_8))); publisher.subscribe(subscriber); @@ -61,7 +62,7 @@ public void checksum_withMultipleChecksums_shouldComputeCorrectChecksums() { String expectedCrc32Digest = "4ac37ece"; String expectedCrc64Digest = "7c05fe704e3e02bc"; - SdkChecksum sha256Checksum = new Sha256Checksum(); + SdkChecksum sha256Checksum = new DigestAlgorithmChecksum(DigestAlgorithm.SHA256); SdkChecksum crc32Checksum = new Crc32Checksum(); SdkChecksum crc64NvmeChecksum = new Crc64NvmeChecksum(); ChecksumSubscriber subscriber = new ChecksumSubscriber(Arrays.asList(sha256Checksum, crc32Checksum, crc64NvmeChecksum)); diff --git a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtilTest.java b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtilTest.java index f3eb940f1bf3..88f220ba83d2 100644 --- a/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtilTest.java +++ b/core/http-auth-aws/src/test/java/software/amazon/awssdk/http/auth/aws/internal/signer/util/ChecksumUtilTest.java @@ -18,7 +18,6 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertThrows; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC32C; import static software.amazon.awssdk.checksums.DefaultChecksumAlgorithm.CRC64NVME; @@ -38,9 +37,7 @@ import software.amazon.awssdk.checksums.SdkChecksum; import software.amazon.awssdk.checksums.internal.Crc32Checksum; import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum; -import software.amazon.awssdk.checksums.internal.Md5Checksum; -import software.amazon.awssdk.checksums.internal.Sha1Checksum; -import software.amazon.awssdk.checksums.internal.Sha256Checksum; +import software.amazon.awssdk.checksums.internal.DigestAlgorithmChecksum; public class ChecksumUtilTest { @@ -56,12 +53,11 @@ public void checksumHeaderName_shouldFormatName() { @Test public void fromChecksumAlgorithm_mapsToCorrectSdkChecksum() { - assertEquals(Sha256Checksum.class, fromChecksumAlgorithm(SHA256).getClass()); - assertEquals(Sha1Checksum.class, fromChecksumAlgorithm(SHA1).getClass()); + assertEquals(DigestAlgorithmChecksum.class, fromChecksumAlgorithm(SHA256).getClass()); + assertEquals(DigestAlgorithmChecksum.class, fromChecksumAlgorithm(SHA1).getClass()); assertEquals(Crc32Checksum.class, fromChecksumAlgorithm(CRC32).getClass()); assertInstanceOf(SdkChecksum.class, fromChecksumAlgorithm(CRC32C)); - assertEquals(Md5Checksum.class, fromChecksumAlgorithm(MD5).getClass()); - assertEquals(Md5Checksum.class, fromChecksumAlgorithm(MD5).getClass()); + assertEquals(DigestAlgorithmChecksum.class, fromChecksumAlgorithm(MD5).getClass()); assertEquals(Crc64NvmeChecksum.class, fromChecksumAlgorithm(CRC64NVME).getClass()); } diff --git a/test/architecture-tests/archunit_store/18fc8858-1308-4d5d-b92d-87817d2fab53 b/test/architecture-tests/archunit_store/18fc8858-1308-4d5d-b92d-87817d2fab53 index 0e7fc4de658f..8ead207e446a 100644 --- a/test/architecture-tests/archunit_store/18fc8858-1308-4d5d-b92d-87817d2fab53 +++ b/test/architecture-tests/archunit_store/18fc8858-1308-4d5d-b92d-87817d2fab53 @@ -50,7 +50,6 @@ Class depends on a Class depends on an internal API from a different module (Class ) Class depends on an internal API from a different module (Class ) Class depends on an internal API from a different module (Class ) -Class depends on an internal API from a different module (Class ) Class depends on an internal API from a different module (Class ) Class depends on an internal API from a different module (Class ) Class depends on an internal API from a different module (Class )