From 27f0a9613cd5744a914cd446cbd223a092fd2e63 Mon Sep 17 00:00:00 2001 From: Max Eliaser Date: Mon, 8 May 2023 14:22:54 -0700 Subject: [PATCH 1/3] Run TransferTests (TransferManager tests) in both HTTPS and HTTP modes. --- .../TransferTests.cpp | 215 ++++++++++-------- 1 file changed, 125 insertions(+), 90 deletions(-) diff --git a/tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp b/tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp index e745d03a939..ebf4e075a7c 100644 --- a/tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp +++ b/tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp @@ -136,8 +136,10 @@ class ScopedTestFile class MockS3Client : public S3Client { public: - MockS3Client(const Aws::Client::ClientConfiguration& clientConfiguration = Aws::Client::ClientConfiguration()): - S3Client(clientConfiguration), executor(clientConfiguration.executor), listObjectsV2RequestCount(0) + MockS3Client(const Aws::Client::ClientConfiguration& clientConfiguration, + std::shared_ptr endpointProvider): + S3Client(clientConfiguration, endpointProvider), + executor(clientConfiguration.executor), listObjectsV2RequestCount(0) {} ~MockS3Client() @@ -161,14 +163,52 @@ class MockS3Client : public S3Client mutable std::atomic listObjectsV2RequestCount; }; -class TransferTests : public ::testing::Test +class ForceHttpEndpointProvider : public Aws::S3::Endpoint::S3EndpointProvider { public: + ForceHttpEndpointProvider() + : S3EndpointProvider() + {} + + Aws::Endpoint::ResolveEndpointOutcome ResolveEndpoint(const Aws::Endpoint::EndpointParameters& endpointParameters) const override + { + auto httpsOutcome = Aws::S3::Endpoint::S3EndpointProvider::ResolveEndpoint(endpointParameters); + if (!httpsOutcome.IsSuccess()) + { + return httpsOutcome; + } + auto endpoint = httpsOutcome.GetResult(); + auto uri = endpoint.GetURI(); + uri.SetScheme(Aws::Http::Scheme::HTTP); + uri.SetPort(80); + endpoint.SetURI(uri); + return Aws::Endpoint::ResolveEndpointOutcome(std::move(endpoint)); + } +}; + +enum class TestType +{ + Https, + Http, +}; +static const std::initializer_list AllTestTypes +{ + TestType::Https, + TestType::Http, +}; - static std::shared_ptr m_s3Client; +class TransferTests : public ::testing::TestWithParam +{ +public: + + static std::map> m_s3Clients; void SetUp() { m_executor = Aws::MakeShared(ALLOCATION_TAG, 4); + if (EmptyBucket(GetTestBucketName())) + { + WaitForBucketToEmpty(GetTestBucketName()); + } } void TearDown() @@ -612,7 +652,10 @@ class TransferTests : public ::testing::Test config.region = AWS_TEST_REGION; // executor used for s3Client config.executor = Aws::MakeShared(ALLOCATION_TAG, 5); - m_s3Client = Aws::MakeShared(ALLOCATION_TAG, config); + m_s3Clients[TestType::Https] = Aws::MakeShared(ALLOCATION_TAG, config, + Aws::MakeShared(ALLOCATION_TAG)); + m_s3Clients[TestType::Http] = Aws::MakeShared(ALLOCATION_TAG, config, + Aws::MakeShared(ALLOCATION_TAG)); CreateBucketRequest createBucket; createBucket.WithBucket(GetTestBucketName()) @@ -625,7 +668,7 @@ class TransferTests : public ::testing::Test createBucket.WithCreateBucketConfiguration(createBucketConfiguration); } - auto createBucketOutcome = m_s3Client->CreateBucket(createBucket); + auto createBucketOutcome = m_s3Clients[TestType::Https]->CreateBucket(createBucket); AWS_ASSERT_SUCCESS(createBucketOutcome); ASSERT_TRUE(WaitForBucketToPropagate(GetTestBucketName())); @@ -642,7 +685,7 @@ class TransferTests : public ::testing::Test { ListObjectsRequest listObjectsRequest; listObjectsRequest.SetBucket(bucketName); - ListObjectsOutcome listObjectsOutcome = m_s3Client->ListObjects(listObjectsRequest); + ListObjectsOutcome listObjectsOutcome = m_s3Clients[TestType::Https]->ListObjects(listObjectsRequest); if (listObjectsOutcome.IsSuccess()) { return true; @@ -662,7 +705,7 @@ class TransferTests : public ::testing::Test GetObjectRequest getObjectRequest; getObjectRequest.SetBucket(bucketName); getObjectRequest.SetKey(objectKey); - GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest); + GetObjectOutcome getObjectOutcome = m_s3Clients[TestType::Https]->GetObject(getObjectRequest); if (getObjectOutcome.IsSuccess()) { return true; @@ -683,7 +726,7 @@ class TransferTests : public ::testing::Test Aws::Vector versions; while(!done) { - ListObjectVersionsOutcome listObjectVersionsOutcome = m_s3Client->ListObjectVersions(listObjectVersionsRequest); + ListObjectVersionsOutcome listObjectVersionsOutcome = m_s3Clients[TestType::Https]->ListObjectVersions(listObjectVersionsRequest); if (!listObjectVersionsOutcome.IsSuccess()) return false; @@ -711,7 +754,7 @@ class TransferTests : public ::testing::Test { deleteObjectRequest.SetVersionId(objectVersion.GetVersionId()); } - m_s3Client->DeleteObject(deleteObjectRequest); + m_s3Clients[TestType::Https]->DeleteObject(deleteObjectRequest); } return true; @@ -725,7 +768,7 @@ class TransferTests : public ::testing::Test unsigned checkForObjectsCount = 0; while (checkForObjectsCount++ < WAIT_MAX_RETRIES) { - ListObjectsOutcome listObjectsOutcome = m_s3Client->ListObjects(listObjectsRequest); + ListObjectsOutcome listObjectsOutcome = m_s3Clients[TestType::Https]->ListObjects(listObjectsRequest); if (!listObjectsOutcome.IsSuccess() || listObjectsOutcome.GetResult().GetContents().size() > 0) { @@ -742,15 +785,11 @@ class TransferTests : public ::testing::Test { HeadBucketRequest headBucketRequest; headBucketRequest.SetBucket(bucketName); - HeadBucketOutcome bucketOutcome = m_s3Client->HeadBucket(headBucketRequest); + HeadBucketOutcome bucketOutcome = m_s3Clients[TestType::Https]->HeadBucket(headBucketRequest); if (bucketOutcome.IsSuccess()) { - AbortMultipartUploadRequest abortRequest; - abortRequest.SetBucket(bucketName); - abortRequest.SetKey(BIG_FILE_KEY); - - m_s3Client->AbortMultipartUpload(abortRequest); + AbortMultiPartUpload(bucketName, BIG_FILE_KEY); if (EmptyBucket(bucketName)) { @@ -761,17 +800,20 @@ class TransferTests : public ::testing::Test DeleteBucketRequest deleteBucketRequest; deleteBucketRequest.SetBucket(bucketName); - DeleteBucketOutcome deleteBucketOutcome = m_s3Client->DeleteBucket(deleteBucketRequest); + DeleteBucketOutcome deleteBucketOutcome = m_s3Clients[TestType::Https]->DeleteBucket(deleteBucketRequest); } static void AbortMultiPartUpload(const Aws::String& bucketName, const Aws::String& fileName) { - AbortMultipartUploadRequest abortRequest; + for (const auto i : AllTestTypes) + { + AbortMultipartUploadRequest abortRequest; - abortRequest.SetBucket(bucketName); - abortRequest.SetKey(fileName); + abortRequest.SetBucket(bucketName); + abortRequest.SetKey(fileName); - m_s3Client->AbortMultipartUpload(abortRequest); + m_s3Clients[i]->AbortMultipartUpload(abortRequest); + } } static void TearDownTestCase() @@ -783,20 +825,20 @@ class TransferTests : public ::testing::Test Aws::FileSystem::DeepDeleteDirectory(GetTestFilesDirectory().c_str()); - m_s3Client = nullptr; + m_s3Clients = {}; } }; -std::shared_ptr TransferTests::m_s3Client(nullptr); +std::map> TransferTests::m_s3Clients({}); -TEST_F(TransferTests, TransferManager_ThreadExecutorJoinsAsyncOperations) +TEST_P(TransferTests, TransferManager_ThreadExecutorJoinsAsyncOperations) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String testFileName = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(testFileName, MB5, testString); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; Aws::Utils::Threading::Semaphore ev(0, 1); transferManagerConfig.downloadProgressCallback = [&ev](const TransferManager*, const std::shared_ptr&){ ev.Release(); }; // When httpRequest returns with error, downloadProgressCallback will not be called. @@ -815,7 +857,7 @@ TEST_F(TransferTests, TransferManager_ThreadExecutorJoinsAsyncOperations) ASSERT_EQ(TransferStatus::COMPLETED, downloadHandle->GetStatus()); } -TEST_F(TransferTests, TransferManager_EmptyFileTest) +TEST_P(TransferTests, TransferManager_EmptyFileTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String emptyTestFileName = MakeFilePath(RandomFileName.c_str()); @@ -828,7 +870,7 @@ TEST_F(TransferTests, TransferManager_EmptyFileTest) TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(emptyTestFileName, GetTestBucketName(), RandomFileName, "text/plain", Aws::Map()); @@ -867,7 +909,7 @@ TEST_F(TransferTests, TransferManager_EmptyFileTest) Aws::Map()); } -TEST_F(TransferTests, TransferManager_SmallTest) +TEST_P(TransferTests, TransferManager_SmallTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String smallTestFileName = MakeFilePath(RandomFileName.c_str()); @@ -880,7 +922,7 @@ TEST_F(TransferTests, TransferManager_SmallTest) TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(smallTestFileName, GetTestBucketName(), RandomFileName, "text/plain", Aws::Map()); @@ -912,14 +954,14 @@ TEST_F(TransferTests, TransferManager_SmallTest) Aws::Map()); } -TEST_F(TransferTests, TransferManager_ContentTest) +TEST_P(TransferTests, TransferManager_ContentTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String contentTestFileName = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(contentTestFileName, CONTENT_TEST_FILE_TEXT); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = @@ -958,7 +1000,7 @@ TEST_F(TransferTests, TransferManager_ContentTest) Aws::Map()); } -TEST_F(TransferTests, TransferManager_DirectoryUploadAndDownloadTest) +TEST_P(TransferTests, TransferManager_DirectoryUploadAndDownloadTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); auto uploadDir = Aws::FileSystem::Join(GetTestFilesDirectory(), RandomFileName + "dirUpload"); @@ -1006,7 +1048,7 @@ TEST_F(TransferTests, TransferManager_DirectoryUploadAndDownloadTest) }; TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; transferManagerConfig.transferInitiatedCallback = transferInitCallback; auto transferManager = TransferManager::Create(transferManagerConfig); @@ -1068,18 +1110,18 @@ TEST_F(TransferTests, TransferManager_DirectoryUploadAndDownloadTest) ASSERT_EQ(uploadTree, downloadTree); // Verify that the updated DownloadToDirectory function only trigger ListObjectsV2Requst once - ASSERT_EQ(1u, m_s3Client->listObjectsV2RequestCount); + ASSERT_EQ(1u, m_s3Clients[GetParam()]->listObjectsV2RequestCount); } // Test of a basic multi part upload - 7.5 megs -TEST_F(TransferTests, TransferManager_MediumTest) +TEST_P(TransferTests, TransferManager_MediumTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String mediumTestFilePath = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(mediumTestFilePath, MEDIUM_TEST_SIZE, testString); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); @@ -1121,14 +1163,14 @@ TEST_F(TransferTests, TransferManager_MediumTest) Aws::Map()); } -TEST_F(TransferTests, TransferManager_BigTest) +TEST_P(TransferTests, TransferManager_BigTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String bigTestFileName = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(bigTestFileName, BIG_TEST_SIZE, testString); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(bigTestFileName, GetTestBucketName(), BIG_FILE_KEY, "text/plain", Aws::Map()); @@ -1176,7 +1218,7 @@ TEST_F(TransferTests, TransferManager_BigTest) Aws::Map()); } -TEST_F(TransferTests, TransferManager_LargeTestDontCare) +TEST_P(TransferTests, TransferManager_LargeTestDontCare) { // Upload a large file, do not check for anything except memory issues. const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); @@ -1184,7 +1226,7 @@ TEST_F(TransferTests, TransferManager_LargeTestDontCare) ScopedTestFile testFile(largeTestFileName, LARGE_TEST_SIZE, testString); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); transferManager->UploadFile(largeTestFileName, GetTestBucketName(), LARGE_FILE_KEY, "text/plain", Aws::Map()); @@ -1196,7 +1238,7 @@ TEST_F(TransferTests, TransferManager_LargeTestDontCare) ASSERT_EQ(Aws::Transfer::TransferStatus::COMPLETED, res); } -TEST_F(TransferTests, TransferManager_MultipartTestWithStreamOffset) +TEST_P(TransferTests, TransferManager_MultipartTestWithStreamOffset) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String bigTestFileName = MakeFilePath(RandomFileName.c_str()); @@ -1204,7 +1246,7 @@ TEST_F(TransferTests, TransferManager_MultipartTestWithStreamOffset) ScopedTestFile testFile(bigTestFileName, BIG_TEST_SIZE, testString); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; #ifdef _MSC_VER auto fileStream = Aws::MakeShared(ALLOCATION_TAG, Aws::Utils::StringUtils::ToWString(bigTestFileName.c_str()).c_str(), std::ios_base::in | std::ios_base::binary); @@ -1255,14 +1297,14 @@ TEST_F(TransferTests, TransferManager_MultipartTestWithStreamOffset) } #ifdef _MSC_VER -TEST_F(TransferTests, TransferManager_UnicodeFileNameTest) +TEST_P(TransferTests, TransferManager_UnicodeFileNameTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String fileName = MakeFilePath((RandomFileName + Aws::Utils::StringUtils::FromWString(UNICODE_TEST_FILE_NAME)).c_str()); ScopedTestFile testFile(fileName, MEDIUM_TEST_SIZE, testString); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(fileName, GetTestBucketName(), UNICODE_FILE_KEY, "text/plain", Aws::Map()); @@ -1304,7 +1346,7 @@ TEST_F(TransferTests, TransferManager_UnicodeFileNameTest) } #endif -TEST_F(TransferTests, TransferManager_CancelAndRetryUploadTest) +TEST_P(TransferTests, TransferManager_CancelAndRetryUploadTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String cancelTestFileName = MakeFilePath(RandomFileName.c_str()); @@ -1341,7 +1383,7 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryUploadTest) } }; - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(cancelTestFileName, GetTestBucketName(), CANCEL_FILE_KEY, "text/plain", Aws::Map(), @@ -1366,7 +1408,7 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryUploadTest) ListMultipartUploadsRequest listMultipartRequest; listMultipartRequest.WithBucket(GetTestBucketName()); - ListMultipartUploadsOutcome listMultipartOutcome = m_s3Client->ListMultipartUploads(listMultipartRequest); + ListMultipartUploadsOutcome listMultipartOutcome = m_s3Clients[GetParam()]->ListMultipartUploads(listMultipartRequest); AWS_EXPECT_SUCCESS(listMultipartOutcome); ASSERT_EQ(1u, listMultipartOutcome.GetResult().GetUploads().size()); @@ -1374,7 +1416,7 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryUploadTest) headObjectRequest.WithBucket(GetTestBucketName()) .WithKey(CANCEL_FILE_KEY); - ASSERT_FALSE(m_s3Client->HeadObject(headObjectRequest).IsSuccess()); + ASSERT_FALSE(m_s3Clients[GetParam()]->HeadObject(headObjectRequest).IsSuccess()); retryInProgress = true; requestPtr = transferManager->RetryUpload(cancelTestFileName, requestPtr); @@ -1396,7 +1438,7 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryUploadTest) ASSERT_EQ(fileSize, requestPtr->GetBytesTransferred()); - listMultipartOutcome = m_s3Client->ListMultipartUploads(listMultipartRequest); + listMultipartOutcome = m_s3Clients[GetParam()]->ListMultipartUploads(listMultipartRequest); AWS_EXPECT_SUCCESS(listMultipartOutcome); ASSERT_EQ(0u, listMultipartOutcome.GetResult().GetUploads().size()); @@ -1411,7 +1453,7 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryUploadTest) Aws::Map()); } -TEST_F(TransferTests, TransferManager_AbortAndRetryUploadTest) +TEST_P(TransferTests, TransferManager_AbortAndRetryUploadTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String cancelTestFileName = MakeFilePath(RandomFileName.c_str()); @@ -1444,7 +1486,7 @@ TEST_F(TransferTests, TransferManager_AbortAndRetryUploadTest) } }; - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); requestPtr = transferManager->UploadFile(cancelTestFileName, GetTestBucketName(), CANCEL_FILE_KEY, "text/plain", Aws::Map()); @@ -1469,7 +1511,7 @@ TEST_F(TransferTests, TransferManager_AbortAndRetryUploadTest) ListMultipartUploadsRequest listMultipartRequest; listMultipartRequest.WithBucket(GetTestBucketName()); - ListMultipartUploadsOutcome listMultipartOutcome = m_s3Client->ListMultipartUploads(listMultipartRequest); + ListMultipartUploadsOutcome listMultipartOutcome = m_s3Clients[GetParam()]->ListMultipartUploads(listMultipartRequest); AWS_EXPECT_SUCCESS(listMultipartOutcome); // S3 has eventual consistency, even thought we called AbortMultiPartUpload and get successful return, @@ -1478,7 +1520,7 @@ TEST_F(TransferTests, TransferManager_AbortAndRetryUploadTest) while (listMultipartOutcome.GetResult().GetUploads().size() != 0u && retries++ < 5) { std::this_thread::sleep_for(std::chrono::seconds(1)); - listMultipartOutcome = m_s3Client->ListMultipartUploads(listMultipartRequest); + listMultipartOutcome = m_s3Clients[GetParam()]->ListMultipartUploads(listMultipartRequest); AWS_EXPECT_SUCCESS(listMultipartOutcome); } ASSERT_EQ(0u, listMultipartOutcome.GetResult().GetUploads().size()); @@ -1513,14 +1555,14 @@ TEST_F(TransferTests, TransferManager_AbortAndRetryUploadTest) Aws::Map()); } -TEST_F(TransferTests, TransferManager_MultiPartContentTest) +TEST_P(TransferTests, TransferManager_MultiPartContentTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String multiPartContentFileName = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(multiPartContentFileName, MEDIUM_TEST_SIZE, MULTI_PART_CONTENT_TEXT); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(multiPartContentFileName, GetTestBucketName(), MULTI_PART_CONTENT_KEY, "text/plain", Aws::Map()); @@ -1556,14 +1598,14 @@ TEST_F(TransferTests, TransferManager_MultiPartContentTest) // Due the racy nature of reporting streamable bytes, only test a completed download // which should return streamable bytes the same as file size. -TEST_F(TransferTests, TransferManager_MultiPartStreamableByteTest) +TEST_P(TransferTests, TransferManager_MultiPartStreamableByteTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String multiPartContentFileName = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(multiPartContentFileName, MEDIUM_TEST_SIZE, MULTI_PART_CONTENT_TEXT); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; transferManagerConfig.transferStatusUpdatedCallback = [&](const TransferManager*, const std::shared_ptr& handle) { if (handle->GetTransferDirection() == TransferDirection::UPLOAD) @@ -1609,7 +1651,7 @@ TEST_F(TransferTests, TransferManager_MultiPartStreamableByteTest) // Single part upload with metadata specified -TEST_F(TransferTests, TransferManager_SinglePartUploadWithMetadataTest) +TEST_P(TransferTests, TransferManager_SinglePartUploadWithMetadataTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String testFilePath = MakeFilePath(RandomFileName.c_str()); @@ -1620,7 +1662,7 @@ TEST_F(TransferTests, TransferManager_SinglePartUploadWithMetadataTest) metadata["key2"] = "val2"; TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(testFilePath, GetTestBucketName(), RandomFileName, "text/plain", metadata); @@ -1636,7 +1678,7 @@ TEST_F(TransferTests, TransferManager_SinglePartUploadWithMetadataTest) headObjectRequest.SetBucket(GetTestBucketName()); headObjectRequest.SetKey(RandomFileName); - HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest); + HeadObjectOutcome headObjectOutcome = m_s3Clients[GetParam()]->HeadObject(headObjectRequest); AWS_ASSERT_SUCCESS(headObjectOutcome); Aws::Map headObjectMetadata = headObjectOutcome.GetResult().GetMetadata(); @@ -1653,7 +1695,7 @@ TEST_F(TransferTests, TransferManager_SinglePartUploadWithMetadataTest) } // Multipart upload with metadata specified -TEST_F(TransferTests, MultipartUploadWithMetadataTest) +TEST_P(TransferTests, MultipartUploadWithMetadataTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String mediumTestFileName = MakeFilePath(RandomFileName.c_str()); @@ -1664,7 +1706,7 @@ TEST_F(TransferTests, MultipartUploadWithMetadataTest) metadata["key2"] = "val2"; TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(mediumTestFileName, GetTestBucketName(), RandomFileName, "text/plain", metadata); @@ -1688,7 +1730,7 @@ TEST_F(TransferTests, MultipartUploadWithMetadataTest) headObjectRequest.SetBucket(GetTestBucketName()); headObjectRequest.SetKey(RandomFileName); - HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest); + HeadObjectOutcome headObjectOutcome = m_s3Clients[GetParam()]->HeadObject(headObjectRequest); AWS_ASSERT_SUCCESS(headObjectOutcome); Aws::Map headObjectMetadata = headObjectOutcome.GetResult().GetMetadata(); @@ -1704,12 +1746,12 @@ TEST_F(TransferTests, MultipartUploadWithMetadataTest) metadata); } -TEST_F(TransferTests, BadFileTest) +TEST_P(TransferTests, BadFileTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String nonsenseFileName = MakeFilePath(RandomFileName.c_str()); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; transferManagerConfig.transferStatusUpdatedCallback = [&](const TransferManager*, const std::shared_ptr& handle) { @@ -1732,7 +1774,7 @@ TEST_F(TransferTests, BadFileTest) ASSERT_EQ(TransferStatus::FAILED, requestPtr->GetStatus()); } -TEST_F(TransferTests, TransferManager_CancelAndRetryDownloadTest) +TEST_P(TransferTests, TransferManager_CancelAndRetryDownloadTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String cancelTestFileName = MakeFilePath(RandomFileName.c_str()); @@ -1741,7 +1783,7 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryDownloadTest) { TransferManagerConfiguration uploadConfig(m_executor.get()); - uploadConfig.s3Client = m_s3Client; + uploadConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(uploadConfig); std::shared_ptr requestPtr = transferManager->UploadFile(cancelTestFileName, GetTestBucketName(), CANCEL_FILE_KEY, "text/plain", Aws::Map()); @@ -1761,7 +1803,7 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryDownloadTest) std::atomic cancelHasBeenCalled(false); TransferManagerConfiguration downloadConfig(m_executor.get()); - downloadConfig.s3Client = m_s3Client; + downloadConfig.s3Client = m_s3Clients[GetParam()]; downloadConfig.transferStatusUpdatedCallback = [&](const TransferManager*, const std::shared_ptr& handle) { @@ -1834,26 +1876,21 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryDownloadTest) } // Single part upload with compute content MD5 -TEST_F(TransferTests, TransferManager_SinglePartUploadWithComputeContentMd5Test) +TEST_P(TransferTests, TransferManager_SinglePartUploadWithComputeContentMd5Test) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String testFilePath = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(testFilePath, MB5, testString); - if (EmptyBucket(GetTestBucketName())) - { - WaitForBucketToEmpty(GetTestBucketName()); - } - GetObjectRequest getObjectRequest; getObjectRequest.SetBucket(GetTestBucketName()); getObjectRequest.SetKey(RandomFileName); - GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest); + GetObjectOutcome getObjectOutcome = m_s3Clients[GetParam()]->GetObject(getObjectRequest); EXPECT_FALSE(getObjectOutcome.IsSuccess()); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; transferManagerConfig.computeContentMD5 = true; auto transferManager = TransferManager::Create(transferManagerConfig); @@ -1870,7 +1907,7 @@ TEST_F(TransferTests, TransferManager_SinglePartUploadWithComputeContentMd5Test) headObjectRequest.SetBucket(GetTestBucketName()); headObjectRequest.SetKey(RandomFileName); - HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest); + HeadObjectOutcome headObjectOutcome = m_s3Clients[GetParam()]->HeadObject(headObjectRequest); AWS_ASSERT_SUCCESS(headObjectOutcome); VerifyUploadedFile(*transferManager, @@ -1882,22 +1919,17 @@ TEST_F(TransferTests, TransferManager_SinglePartUploadWithComputeContentMd5Test) } // Multipart upload with compute content MD5 -TEST_F(TransferTests, MultipartUploadWithComputeContentMd5Test) +TEST_P(TransferTests, MultipartUploadWithComputeContentMd5Test) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String testFilePath = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(testFilePath, MEDIUM_TEST_SIZE, testString); - if (EmptyBucket(GetTestBucketName())) - { - WaitForBucketToEmpty(GetTestBucketName()); - } - GetObjectRequest getObjectRequest; getObjectRequest.SetBucket(GetTestBucketName()); getObjectRequest.SetKey(RandomFileName); - GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest); + GetObjectOutcome getObjectOutcome = m_s3Clients[GetParam()]->GetObject(getObjectRequest); EXPECT_FALSE(getObjectOutcome.IsSuccess()); ListMultipartUploadsRequest listMultipartRequest; @@ -1905,7 +1937,7 @@ TEST_F(TransferTests, MultipartUploadWithComputeContentMd5Test) listMultipartRequest.SetBucket(GetTestBucketName()); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; transferManagerConfig.computeContentMD5 = true; auto transferManager = TransferManager::Create(transferManagerConfig); @@ -1931,7 +1963,7 @@ TEST_F(TransferTests, MultipartUploadWithComputeContentMd5Test) headObjectRequest.SetBucket(GetTestBucketName()); headObjectRequest.SetKey(RandomFileName); - HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest); + HeadObjectOutcome headObjectOutcome = m_s3Clients[GetParam()]->HeadObject(headObjectRequest); AWS_ASSERT_SUCCESS(headObjectOutcome); VerifyUploadedFile(*transferManager, @@ -1943,7 +1975,7 @@ TEST_F(TransferTests, MultipartUploadWithComputeContentMd5Test) } // Test of a basic multi part upload - 7.5 megs with the templated tests -TEST_F(TransferTests, TransferManager_TemplatesTest) +TEST_P(TransferTests, TransferManager_TemplatesTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String mediumTestFilePath = MakeFilePath(RandomFileName.c_str()); @@ -1956,7 +1988,7 @@ TEST_F(TransferTests, TransferManager_TemplatesTest) putObjReqTemplate.WithRequestPayer(Aws::S3::Model::RequestPayer::requester); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; transferManagerConfig.putObjectTemplate = putObjReqTemplate; transferManagerConfig.getObjectTemplate = getObjReqTemplate; @@ -2018,7 +2050,7 @@ class TestHelperTransferManager : public Aws::Transfer::TransferManager }; }; -TEST_F(TransferTests, TransferManager_TestRelativePrefix) +TEST_P(TransferTests, TransferManager_TestRelativePrefix) { static const std::vector TEST_CASES = { @@ -2078,4 +2110,7 @@ TEST_F(TransferTests, TransferManager_TestRelativePrefix) } } +INSTANTIATE_TEST_SUITE_P(Https, TransferTests, testing::Values(TestType::Https)); +INSTANTIATE_TEST_SUITE_P(Http, TransferTests, testing::Values(TestType::Http)); + } From e404f1f5afadce83fc3a832275ce8fbd7d7474de Mon Sep 17 00:00:00 2001 From: Max Eliaser Date: Mon, 8 May 2023 16:33:34 -0700 Subject: [PATCH 2/3] Move TransferManager_EmptyFileTest to the beginning of TransferTests suite For whatever reason, if another HTTP-mode test runs before this one, it won't reproduce the problem (running the HTTPS-mode tests first is okay.) Without reordering the tests, I have to run only this one test case using --gtest_filter in order for it to reproduce the problem. --- .../TransferTests.cpp | 54 ++++++++++--------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp b/tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp index ebf4e075a7c..e17442ed3d4 100644 --- a/tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp +++ b/tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp @@ -831,32 +831,8 @@ class TransferTests : public ::testing::TestWithParam std::map> TransferTests::m_s3Clients({}); -TEST_P(TransferTests, TransferManager_ThreadExecutorJoinsAsyncOperations) -{ - const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); - Aws::String testFileName = MakeFilePath(RandomFileName.c_str()); - - ScopedTestFile testFile(testFileName, MB5, testString); - TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Clients[GetParam()]; - Aws::Utils::Threading::Semaphore ev(0, 1); - transferManagerConfig.downloadProgressCallback = [&ev](const TransferManager*, const std::shared_ptr&){ ev.Release(); }; - // When httpRequest returns with error, downloadProgressCallback will not be called. - transferManagerConfig.errorCallback = [&ev](const TransferManager*, const std::shared_ptr&, const Aws::Client::AWSError&){ ev.Release(); }; - std::shared_ptr uploadHandle, downloadHandle; - { - auto transferManager = TransferManager::Create(transferManagerConfig); - uploadHandle = transferManager->UploadFile(testFileName, GetTestBucketName(), RandomFileName, "text/plain", Aws::Map()); - uploadHandle->WaitUntilFinished(); - ASSERT_TRUE(WaitForObjectToPropagate(GetTestBucketName(), RandomFileName.c_str())); - downloadHandle = transferManager->DownloadFile(GetTestBucketName(), RandomFileName, MakeDownloadFileName(RandomFileName)); - } - ev.WaitOne(); // ensures that the download has started; otherwise, downloadHandle's status will be NOT_STARTED - m_executor = nullptr; // this should join all worker threads. - ASSERT_EQ(TransferStatus::COMPLETED, uploadHandle->GetStatus()); - ASSERT_EQ(TransferStatus::COMPLETED, downloadHandle->GetStatus()); -} - +// NOTE: this has to be the first test in the suite in order to reproduce the +// CurlCode 56 problem in HTTP mode reliably TEST_P(TransferTests, TransferManager_EmptyFileTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); @@ -909,6 +885,32 @@ TEST_P(TransferTests, TransferManager_EmptyFileTest) Aws::Map()); } +TEST_P(TransferTests, TransferManager_ThreadExecutorJoinsAsyncOperations) +{ + const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); + Aws::String testFileName = MakeFilePath(RandomFileName.c_str()); + + ScopedTestFile testFile(testFileName, MB5, testString); + TransferManagerConfiguration transferManagerConfig(m_executor.get()); + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; + Aws::Utils::Threading::Semaphore ev(0, 1); + transferManagerConfig.downloadProgressCallback = [&ev](const TransferManager*, const std::shared_ptr&){ ev.Release(); }; + // When httpRequest returns with error, downloadProgressCallback will not be called. + transferManagerConfig.errorCallback = [&ev](const TransferManager*, const std::shared_ptr&, const Aws::Client::AWSError&){ ev.Release(); }; + std::shared_ptr uploadHandle, downloadHandle; + { + auto transferManager = TransferManager::Create(transferManagerConfig); + uploadHandle = transferManager->UploadFile(testFileName, GetTestBucketName(), RandomFileName, "text/plain", Aws::Map()); + uploadHandle->WaitUntilFinished(); + ASSERT_TRUE(WaitForObjectToPropagate(GetTestBucketName(), RandomFileName.c_str())); + downloadHandle = transferManager->DownloadFile(GetTestBucketName(), RandomFileName, MakeDownloadFileName(RandomFileName)); + } + ev.WaitOne(); // ensures that the download has started; otherwise, downloadHandle's status will be NOT_STARTED + m_executor = nullptr; // this should join all worker threads. + ASSERT_EQ(TransferStatus::COMPLETED, uploadHandle->GetStatus()); + ASSERT_EQ(TransferStatus::COMPLETED, downloadHandle->GetStatus()); +} + TEST_P(TransferTests, TransferManager_SmallTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); From 4e9824cfcaaa42f0f86ef74ee6727c1f3883fec6 Mon Sep 17 00:00:00 2001 From: Max Eliaser Date: Mon, 8 May 2023 16:49:38 -0700 Subject: [PATCH 3/3] TransferManager: don't set byte range header for 0-byte DoSinglePartDownload Fixes an issue similar to #738, but which only shows up when using HTTPS endpoints. --- .../source/transfer/TransferManager.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp b/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp index aa566d1adca..30073d0057b 100644 --- a/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp +++ b/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp @@ -758,10 +758,13 @@ namespace Aws auto request = m_transferConfig.getObjectTemplate; request.SetCustomizedAccessLogTag(m_transferConfig.customizedAccessLogTag); request.SetContinueRequestHandler([handle](const Aws::Http::HttpRequest*) { return handle->ShouldContinue(); }); - request.SetRange( - FormatRangeSpecifier( - handle->GetBytesOffset(), - handle->GetBytesOffset() + handle->GetBytesTotalSize() - 1)); + if (handle->GetBytesTotalSize() != 0) + { + request.SetRange( + FormatRangeSpecifier( + handle->GetBytesOffset(), + handle->GetBytesOffset() + handle->GetBytesTotalSize() - 1)); + } request.WithBucket(handle->GetBucketName()) .WithKey(handle->GetKey());