diff --git a/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp b/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp index aa566d1adca..30073d0057b 100644 --- a/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp +++ b/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp @@ -758,10 +758,13 @@ namespace Aws auto request = m_transferConfig.getObjectTemplate; request.SetCustomizedAccessLogTag(m_transferConfig.customizedAccessLogTag); request.SetContinueRequestHandler([handle](const Aws::Http::HttpRequest*) { return handle->ShouldContinue(); }); - request.SetRange( - FormatRangeSpecifier( - handle->GetBytesOffset(), - handle->GetBytesOffset() + handle->GetBytesTotalSize() - 1)); + if (handle->GetBytesTotalSize() != 0) + { + request.SetRange( + FormatRangeSpecifier( + handle->GetBytesOffset(), + handle->GetBytesOffset() + handle->GetBytesTotalSize() - 1)); + } request.WithBucket(handle->GetBucketName()) .WithKey(handle->GetKey()); diff --git a/tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp b/tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp index e745d03a939..e17442ed3d4 100644 --- a/tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp +++ b/tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp @@ -136,8 +136,10 @@ class ScopedTestFile class MockS3Client : public S3Client { public: - MockS3Client(const Aws::Client::ClientConfiguration& clientConfiguration = Aws::Client::ClientConfiguration()): - S3Client(clientConfiguration), executor(clientConfiguration.executor), listObjectsV2RequestCount(0) + MockS3Client(const Aws::Client::ClientConfiguration& clientConfiguration, + std::shared_ptr endpointProvider): + S3Client(clientConfiguration, endpointProvider), + executor(clientConfiguration.executor), listObjectsV2RequestCount(0) {} ~MockS3Client() @@ -161,14 +163,52 @@ class MockS3Client : public S3Client mutable std::atomic listObjectsV2RequestCount; }; -class TransferTests : public ::testing::Test +class ForceHttpEndpointProvider : public Aws::S3::Endpoint::S3EndpointProvider +{ +public: + ForceHttpEndpointProvider() + : S3EndpointProvider() + {} + + Aws::Endpoint::ResolveEndpointOutcome ResolveEndpoint(const Aws::Endpoint::EndpointParameters& endpointParameters) const override + { + auto httpsOutcome = Aws::S3::Endpoint::S3EndpointProvider::ResolveEndpoint(endpointParameters); + if (!httpsOutcome.IsSuccess()) + { + return httpsOutcome; + } + auto endpoint = httpsOutcome.GetResult(); + auto uri = endpoint.GetURI(); + uri.SetScheme(Aws::Http::Scheme::HTTP); + uri.SetPort(80); + endpoint.SetURI(uri); + return Aws::Endpoint::ResolveEndpointOutcome(std::move(endpoint)); + } +}; + +enum class TestType +{ + Https, + Http, +}; +static const std::initializer_list AllTestTypes +{ + TestType::Https, + TestType::Http, +}; + +class TransferTests : public ::testing::TestWithParam { public: - static std::shared_ptr m_s3Client; + static std::map> m_s3Clients; void SetUp() { m_executor = Aws::MakeShared(ALLOCATION_TAG, 4); + if (EmptyBucket(GetTestBucketName())) + { + WaitForBucketToEmpty(GetTestBucketName()); + } } void TearDown() @@ -612,7 +652,10 @@ class TransferTests : public ::testing::Test config.region = AWS_TEST_REGION; // executor used for s3Client config.executor = Aws::MakeShared(ALLOCATION_TAG, 5); - m_s3Client = Aws::MakeShared(ALLOCATION_TAG, config); + m_s3Clients[TestType::Https] = Aws::MakeShared(ALLOCATION_TAG, config, + Aws::MakeShared(ALLOCATION_TAG)); + m_s3Clients[TestType::Http] = Aws::MakeShared(ALLOCATION_TAG, config, + Aws::MakeShared(ALLOCATION_TAG)); CreateBucketRequest createBucket; createBucket.WithBucket(GetTestBucketName()) @@ -625,7 +668,7 @@ class TransferTests : public ::testing::Test createBucket.WithCreateBucketConfiguration(createBucketConfiguration); } - auto createBucketOutcome = m_s3Client->CreateBucket(createBucket); + auto createBucketOutcome = m_s3Clients[TestType::Https]->CreateBucket(createBucket); AWS_ASSERT_SUCCESS(createBucketOutcome); ASSERT_TRUE(WaitForBucketToPropagate(GetTestBucketName())); @@ -642,7 +685,7 @@ class TransferTests : public ::testing::Test { ListObjectsRequest listObjectsRequest; listObjectsRequest.SetBucket(bucketName); - ListObjectsOutcome listObjectsOutcome = m_s3Client->ListObjects(listObjectsRequest); + ListObjectsOutcome listObjectsOutcome = m_s3Clients[TestType::Https]->ListObjects(listObjectsRequest); if (listObjectsOutcome.IsSuccess()) { return true; @@ -662,7 +705,7 @@ class TransferTests : public ::testing::Test GetObjectRequest getObjectRequest; getObjectRequest.SetBucket(bucketName); getObjectRequest.SetKey(objectKey); - GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest); + GetObjectOutcome getObjectOutcome = m_s3Clients[TestType::Https]->GetObject(getObjectRequest); if (getObjectOutcome.IsSuccess()) { return true; @@ -683,7 +726,7 @@ class TransferTests : public ::testing::Test Aws::Vector versions; while(!done) { - ListObjectVersionsOutcome listObjectVersionsOutcome = m_s3Client->ListObjectVersions(listObjectVersionsRequest); + ListObjectVersionsOutcome listObjectVersionsOutcome = m_s3Clients[TestType::Https]->ListObjectVersions(listObjectVersionsRequest); if (!listObjectVersionsOutcome.IsSuccess()) return false; @@ -711,7 +754,7 @@ class TransferTests : public ::testing::Test { deleteObjectRequest.SetVersionId(objectVersion.GetVersionId()); } - m_s3Client->DeleteObject(deleteObjectRequest); + m_s3Clients[TestType::Https]->DeleteObject(deleteObjectRequest); } return true; @@ -725,7 +768,7 @@ class TransferTests : public ::testing::Test unsigned checkForObjectsCount = 0; while (checkForObjectsCount++ < WAIT_MAX_RETRIES) { - ListObjectsOutcome listObjectsOutcome = m_s3Client->ListObjects(listObjectsRequest); + ListObjectsOutcome listObjectsOutcome = m_s3Clients[TestType::Https]->ListObjects(listObjectsRequest); if (!listObjectsOutcome.IsSuccess() || listObjectsOutcome.GetResult().GetContents().size() > 0) { @@ -742,15 +785,11 @@ class TransferTests : public ::testing::Test { HeadBucketRequest headBucketRequest; headBucketRequest.SetBucket(bucketName); - HeadBucketOutcome bucketOutcome = m_s3Client->HeadBucket(headBucketRequest); + HeadBucketOutcome bucketOutcome = m_s3Clients[TestType::Https]->HeadBucket(headBucketRequest); if (bucketOutcome.IsSuccess()) { - AbortMultipartUploadRequest abortRequest; - abortRequest.SetBucket(bucketName); - abortRequest.SetKey(BIG_FILE_KEY); - - m_s3Client->AbortMultipartUpload(abortRequest); + AbortMultiPartUpload(bucketName, BIG_FILE_KEY); if (EmptyBucket(bucketName)) { @@ -761,17 +800,20 @@ class TransferTests : public ::testing::Test DeleteBucketRequest deleteBucketRequest; deleteBucketRequest.SetBucket(bucketName); - DeleteBucketOutcome deleteBucketOutcome = m_s3Client->DeleteBucket(deleteBucketRequest); + DeleteBucketOutcome deleteBucketOutcome = m_s3Clients[TestType::Https]->DeleteBucket(deleteBucketRequest); } static void AbortMultiPartUpload(const Aws::String& bucketName, const Aws::String& fileName) { - AbortMultipartUploadRequest abortRequest; + for (const auto i : AllTestTypes) + { + AbortMultipartUploadRequest abortRequest; - abortRequest.SetBucket(bucketName); - abortRequest.SetKey(fileName); + abortRequest.SetBucket(bucketName); + abortRequest.SetKey(fileName); - m_s3Client->AbortMultipartUpload(abortRequest); + m_s3Clients[i]->AbortMultipartUpload(abortRequest); + } } static void TearDownTestCase() @@ -783,39 +825,15 @@ class TransferTests : public ::testing::Test Aws::FileSystem::DeepDeleteDirectory(GetTestFilesDirectory().c_str()); - m_s3Client = nullptr; + m_s3Clients = {}; } }; -std::shared_ptr TransferTests::m_s3Client(nullptr); +std::map> TransferTests::m_s3Clients({}); -TEST_F(TransferTests, TransferManager_ThreadExecutorJoinsAsyncOperations) -{ - const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); - Aws::String testFileName = MakeFilePath(RandomFileName.c_str()); - - ScopedTestFile testFile(testFileName, MB5, testString); - TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; - Aws::Utils::Threading::Semaphore ev(0, 1); - transferManagerConfig.downloadProgressCallback = [&ev](const TransferManager*, const std::shared_ptr&){ ev.Release(); }; - // When httpRequest returns with error, downloadProgressCallback will not be called. - transferManagerConfig.errorCallback = [&ev](const TransferManager*, const std::shared_ptr&, const Aws::Client::AWSError&){ ev.Release(); }; - std::shared_ptr uploadHandle, downloadHandle; - { - auto transferManager = TransferManager::Create(transferManagerConfig); - uploadHandle = transferManager->UploadFile(testFileName, GetTestBucketName(), RandomFileName, "text/plain", Aws::Map()); - uploadHandle->WaitUntilFinished(); - ASSERT_TRUE(WaitForObjectToPropagate(GetTestBucketName(), RandomFileName.c_str())); - downloadHandle = transferManager->DownloadFile(GetTestBucketName(), RandomFileName, MakeDownloadFileName(RandomFileName)); - } - ev.WaitOne(); // ensures that the download has started; otherwise, downloadHandle's status will be NOT_STARTED - m_executor = nullptr; // this should join all worker threads. - ASSERT_EQ(TransferStatus::COMPLETED, uploadHandle->GetStatus()); - ASSERT_EQ(TransferStatus::COMPLETED, downloadHandle->GetStatus()); -} - -TEST_F(TransferTests, TransferManager_EmptyFileTest) +// NOTE: this has to be the first test in the suite in order to reproduce the +// CurlCode 56 problem in HTTP mode reliably +TEST_P(TransferTests, TransferManager_EmptyFileTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String emptyTestFileName = MakeFilePath(RandomFileName.c_str()); @@ -828,7 +846,7 @@ TEST_F(TransferTests, TransferManager_EmptyFileTest) TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(emptyTestFileName, GetTestBucketName(), RandomFileName, "text/plain", Aws::Map()); @@ -867,7 +885,33 @@ TEST_F(TransferTests, TransferManager_EmptyFileTest) Aws::Map()); } -TEST_F(TransferTests, TransferManager_SmallTest) +TEST_P(TransferTests, TransferManager_ThreadExecutorJoinsAsyncOperations) +{ + const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); + Aws::String testFileName = MakeFilePath(RandomFileName.c_str()); + + ScopedTestFile testFile(testFileName, MB5, testString); + TransferManagerConfiguration transferManagerConfig(m_executor.get()); + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; + Aws::Utils::Threading::Semaphore ev(0, 1); + transferManagerConfig.downloadProgressCallback = [&ev](const TransferManager*, const std::shared_ptr&){ ev.Release(); }; + // When httpRequest returns with error, downloadProgressCallback will not be called. + transferManagerConfig.errorCallback = [&ev](const TransferManager*, const std::shared_ptr&, const Aws::Client::AWSError&){ ev.Release(); }; + std::shared_ptr uploadHandle, downloadHandle; + { + auto transferManager = TransferManager::Create(transferManagerConfig); + uploadHandle = transferManager->UploadFile(testFileName, GetTestBucketName(), RandomFileName, "text/plain", Aws::Map()); + uploadHandle->WaitUntilFinished(); + ASSERT_TRUE(WaitForObjectToPropagate(GetTestBucketName(), RandomFileName.c_str())); + downloadHandle = transferManager->DownloadFile(GetTestBucketName(), RandomFileName, MakeDownloadFileName(RandomFileName)); + } + ev.WaitOne(); // ensures that the download has started; otherwise, downloadHandle's status will be NOT_STARTED + m_executor = nullptr; // this should join all worker threads. + ASSERT_EQ(TransferStatus::COMPLETED, uploadHandle->GetStatus()); + ASSERT_EQ(TransferStatus::COMPLETED, downloadHandle->GetStatus()); +} + +TEST_P(TransferTests, TransferManager_SmallTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String smallTestFileName = MakeFilePath(RandomFileName.c_str()); @@ -880,7 +924,7 @@ TEST_F(TransferTests, TransferManager_SmallTest) TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(smallTestFileName, GetTestBucketName(), RandomFileName, "text/plain", Aws::Map()); @@ -912,14 +956,14 @@ TEST_F(TransferTests, TransferManager_SmallTest) Aws::Map()); } -TEST_F(TransferTests, TransferManager_ContentTest) +TEST_P(TransferTests, TransferManager_ContentTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String contentTestFileName = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(contentTestFileName, CONTENT_TEST_FILE_TEXT); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = @@ -958,7 +1002,7 @@ TEST_F(TransferTests, TransferManager_ContentTest) Aws::Map()); } -TEST_F(TransferTests, TransferManager_DirectoryUploadAndDownloadTest) +TEST_P(TransferTests, TransferManager_DirectoryUploadAndDownloadTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); auto uploadDir = Aws::FileSystem::Join(GetTestFilesDirectory(), RandomFileName + "dirUpload"); @@ -1006,7 +1050,7 @@ TEST_F(TransferTests, TransferManager_DirectoryUploadAndDownloadTest) }; TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; transferManagerConfig.transferInitiatedCallback = transferInitCallback; auto transferManager = TransferManager::Create(transferManagerConfig); @@ -1068,18 +1112,18 @@ TEST_F(TransferTests, TransferManager_DirectoryUploadAndDownloadTest) ASSERT_EQ(uploadTree, downloadTree); // Verify that the updated DownloadToDirectory function only trigger ListObjectsV2Requst once - ASSERT_EQ(1u, m_s3Client->listObjectsV2RequestCount); + ASSERT_EQ(1u, m_s3Clients[GetParam()]->listObjectsV2RequestCount); } // Test of a basic multi part upload - 7.5 megs -TEST_F(TransferTests, TransferManager_MediumTest) +TEST_P(TransferTests, TransferManager_MediumTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String mediumTestFilePath = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(mediumTestFilePath, MEDIUM_TEST_SIZE, testString); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); @@ -1121,14 +1165,14 @@ TEST_F(TransferTests, TransferManager_MediumTest) Aws::Map()); } -TEST_F(TransferTests, TransferManager_BigTest) +TEST_P(TransferTests, TransferManager_BigTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String bigTestFileName = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(bigTestFileName, BIG_TEST_SIZE, testString); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(bigTestFileName, GetTestBucketName(), BIG_FILE_KEY, "text/plain", Aws::Map()); @@ -1176,7 +1220,7 @@ TEST_F(TransferTests, TransferManager_BigTest) Aws::Map()); } -TEST_F(TransferTests, TransferManager_LargeTestDontCare) +TEST_P(TransferTests, TransferManager_LargeTestDontCare) { // Upload a large file, do not check for anything except memory issues. const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); @@ -1184,7 +1228,7 @@ TEST_F(TransferTests, TransferManager_LargeTestDontCare) ScopedTestFile testFile(largeTestFileName, LARGE_TEST_SIZE, testString); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); transferManager->UploadFile(largeTestFileName, GetTestBucketName(), LARGE_FILE_KEY, "text/plain", Aws::Map()); @@ -1196,7 +1240,7 @@ TEST_F(TransferTests, TransferManager_LargeTestDontCare) ASSERT_EQ(Aws::Transfer::TransferStatus::COMPLETED, res); } -TEST_F(TransferTests, TransferManager_MultipartTestWithStreamOffset) +TEST_P(TransferTests, TransferManager_MultipartTestWithStreamOffset) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String bigTestFileName = MakeFilePath(RandomFileName.c_str()); @@ -1204,7 +1248,7 @@ TEST_F(TransferTests, TransferManager_MultipartTestWithStreamOffset) ScopedTestFile testFile(bigTestFileName, BIG_TEST_SIZE, testString); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; #ifdef _MSC_VER auto fileStream = Aws::MakeShared(ALLOCATION_TAG, Aws::Utils::StringUtils::ToWString(bigTestFileName.c_str()).c_str(), std::ios_base::in | std::ios_base::binary); @@ -1255,14 +1299,14 @@ TEST_F(TransferTests, TransferManager_MultipartTestWithStreamOffset) } #ifdef _MSC_VER -TEST_F(TransferTests, TransferManager_UnicodeFileNameTest) +TEST_P(TransferTests, TransferManager_UnicodeFileNameTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String fileName = MakeFilePath((RandomFileName + Aws::Utils::StringUtils::FromWString(UNICODE_TEST_FILE_NAME)).c_str()); ScopedTestFile testFile(fileName, MEDIUM_TEST_SIZE, testString); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(fileName, GetTestBucketName(), UNICODE_FILE_KEY, "text/plain", Aws::Map()); @@ -1304,7 +1348,7 @@ TEST_F(TransferTests, TransferManager_UnicodeFileNameTest) } #endif -TEST_F(TransferTests, TransferManager_CancelAndRetryUploadTest) +TEST_P(TransferTests, TransferManager_CancelAndRetryUploadTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String cancelTestFileName = MakeFilePath(RandomFileName.c_str()); @@ -1341,7 +1385,7 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryUploadTest) } }; - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(cancelTestFileName, GetTestBucketName(), CANCEL_FILE_KEY, "text/plain", Aws::Map(), @@ -1366,7 +1410,7 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryUploadTest) ListMultipartUploadsRequest listMultipartRequest; listMultipartRequest.WithBucket(GetTestBucketName()); - ListMultipartUploadsOutcome listMultipartOutcome = m_s3Client->ListMultipartUploads(listMultipartRequest); + ListMultipartUploadsOutcome listMultipartOutcome = m_s3Clients[GetParam()]->ListMultipartUploads(listMultipartRequest); AWS_EXPECT_SUCCESS(listMultipartOutcome); ASSERT_EQ(1u, listMultipartOutcome.GetResult().GetUploads().size()); @@ -1374,7 +1418,7 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryUploadTest) headObjectRequest.WithBucket(GetTestBucketName()) .WithKey(CANCEL_FILE_KEY); - ASSERT_FALSE(m_s3Client->HeadObject(headObjectRequest).IsSuccess()); + ASSERT_FALSE(m_s3Clients[GetParam()]->HeadObject(headObjectRequest).IsSuccess()); retryInProgress = true; requestPtr = transferManager->RetryUpload(cancelTestFileName, requestPtr); @@ -1396,7 +1440,7 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryUploadTest) ASSERT_EQ(fileSize, requestPtr->GetBytesTransferred()); - listMultipartOutcome = m_s3Client->ListMultipartUploads(listMultipartRequest); + listMultipartOutcome = m_s3Clients[GetParam()]->ListMultipartUploads(listMultipartRequest); AWS_EXPECT_SUCCESS(listMultipartOutcome); ASSERT_EQ(0u, listMultipartOutcome.GetResult().GetUploads().size()); @@ -1411,7 +1455,7 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryUploadTest) Aws::Map()); } -TEST_F(TransferTests, TransferManager_AbortAndRetryUploadTest) +TEST_P(TransferTests, TransferManager_AbortAndRetryUploadTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String cancelTestFileName = MakeFilePath(RandomFileName.c_str()); @@ -1444,7 +1488,7 @@ TEST_F(TransferTests, TransferManager_AbortAndRetryUploadTest) } }; - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); requestPtr = transferManager->UploadFile(cancelTestFileName, GetTestBucketName(), CANCEL_FILE_KEY, "text/plain", Aws::Map()); @@ -1469,7 +1513,7 @@ TEST_F(TransferTests, TransferManager_AbortAndRetryUploadTest) ListMultipartUploadsRequest listMultipartRequest; listMultipartRequest.WithBucket(GetTestBucketName()); - ListMultipartUploadsOutcome listMultipartOutcome = m_s3Client->ListMultipartUploads(listMultipartRequest); + ListMultipartUploadsOutcome listMultipartOutcome = m_s3Clients[GetParam()]->ListMultipartUploads(listMultipartRequest); AWS_EXPECT_SUCCESS(listMultipartOutcome); // S3 has eventual consistency, even thought we called AbortMultiPartUpload and get successful return, @@ -1478,7 +1522,7 @@ TEST_F(TransferTests, TransferManager_AbortAndRetryUploadTest) while (listMultipartOutcome.GetResult().GetUploads().size() != 0u && retries++ < 5) { std::this_thread::sleep_for(std::chrono::seconds(1)); - listMultipartOutcome = m_s3Client->ListMultipartUploads(listMultipartRequest); + listMultipartOutcome = m_s3Clients[GetParam()]->ListMultipartUploads(listMultipartRequest); AWS_EXPECT_SUCCESS(listMultipartOutcome); } ASSERT_EQ(0u, listMultipartOutcome.GetResult().GetUploads().size()); @@ -1513,14 +1557,14 @@ TEST_F(TransferTests, TransferManager_AbortAndRetryUploadTest) Aws::Map()); } -TEST_F(TransferTests, TransferManager_MultiPartContentTest) +TEST_P(TransferTests, TransferManager_MultiPartContentTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String multiPartContentFileName = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(multiPartContentFileName, MEDIUM_TEST_SIZE, MULTI_PART_CONTENT_TEXT); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(multiPartContentFileName, GetTestBucketName(), MULTI_PART_CONTENT_KEY, "text/plain", Aws::Map()); @@ -1556,14 +1600,14 @@ TEST_F(TransferTests, TransferManager_MultiPartContentTest) // Due the racy nature of reporting streamable bytes, only test a completed download // which should return streamable bytes the same as file size. -TEST_F(TransferTests, TransferManager_MultiPartStreamableByteTest) +TEST_P(TransferTests, TransferManager_MultiPartStreamableByteTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String multiPartContentFileName = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(multiPartContentFileName, MEDIUM_TEST_SIZE, MULTI_PART_CONTENT_TEXT); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; transferManagerConfig.transferStatusUpdatedCallback = [&](const TransferManager*, const std::shared_ptr& handle) { if (handle->GetTransferDirection() == TransferDirection::UPLOAD) @@ -1609,7 +1653,7 @@ TEST_F(TransferTests, TransferManager_MultiPartStreamableByteTest) // Single part upload with metadata specified -TEST_F(TransferTests, TransferManager_SinglePartUploadWithMetadataTest) +TEST_P(TransferTests, TransferManager_SinglePartUploadWithMetadataTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String testFilePath = MakeFilePath(RandomFileName.c_str()); @@ -1620,7 +1664,7 @@ TEST_F(TransferTests, TransferManager_SinglePartUploadWithMetadataTest) metadata["key2"] = "val2"; TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(testFilePath, GetTestBucketName(), RandomFileName, "text/plain", metadata); @@ -1636,7 +1680,7 @@ TEST_F(TransferTests, TransferManager_SinglePartUploadWithMetadataTest) headObjectRequest.SetBucket(GetTestBucketName()); headObjectRequest.SetKey(RandomFileName); - HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest); + HeadObjectOutcome headObjectOutcome = m_s3Clients[GetParam()]->HeadObject(headObjectRequest); AWS_ASSERT_SUCCESS(headObjectOutcome); Aws::Map headObjectMetadata = headObjectOutcome.GetResult().GetMetadata(); @@ -1653,7 +1697,7 @@ TEST_F(TransferTests, TransferManager_SinglePartUploadWithMetadataTest) } // Multipart upload with metadata specified -TEST_F(TransferTests, MultipartUploadWithMetadataTest) +TEST_P(TransferTests, MultipartUploadWithMetadataTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String mediumTestFileName = MakeFilePath(RandomFileName.c_str()); @@ -1664,7 +1708,7 @@ TEST_F(TransferTests, MultipartUploadWithMetadataTest) metadata["key2"] = "val2"; TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(transferManagerConfig); std::shared_ptr requestPtr = transferManager->UploadFile(mediumTestFileName, GetTestBucketName(), RandomFileName, "text/plain", metadata); @@ -1688,7 +1732,7 @@ TEST_F(TransferTests, MultipartUploadWithMetadataTest) headObjectRequest.SetBucket(GetTestBucketName()); headObjectRequest.SetKey(RandomFileName); - HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest); + HeadObjectOutcome headObjectOutcome = m_s3Clients[GetParam()]->HeadObject(headObjectRequest); AWS_ASSERT_SUCCESS(headObjectOutcome); Aws::Map headObjectMetadata = headObjectOutcome.GetResult().GetMetadata(); @@ -1704,12 +1748,12 @@ TEST_F(TransferTests, MultipartUploadWithMetadataTest) metadata); } -TEST_F(TransferTests, BadFileTest) +TEST_P(TransferTests, BadFileTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String nonsenseFileName = MakeFilePath(RandomFileName.c_str()); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; transferManagerConfig.transferStatusUpdatedCallback = [&](const TransferManager*, const std::shared_ptr& handle) { @@ -1732,7 +1776,7 @@ TEST_F(TransferTests, BadFileTest) ASSERT_EQ(TransferStatus::FAILED, requestPtr->GetStatus()); } -TEST_F(TransferTests, TransferManager_CancelAndRetryDownloadTest) +TEST_P(TransferTests, TransferManager_CancelAndRetryDownloadTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String cancelTestFileName = MakeFilePath(RandomFileName.c_str()); @@ -1741,7 +1785,7 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryDownloadTest) { TransferManagerConfiguration uploadConfig(m_executor.get()); - uploadConfig.s3Client = m_s3Client; + uploadConfig.s3Client = m_s3Clients[GetParam()]; auto transferManager = TransferManager::Create(uploadConfig); std::shared_ptr requestPtr = transferManager->UploadFile(cancelTestFileName, GetTestBucketName(), CANCEL_FILE_KEY, "text/plain", Aws::Map()); @@ -1761,7 +1805,7 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryDownloadTest) std::atomic cancelHasBeenCalled(false); TransferManagerConfiguration downloadConfig(m_executor.get()); - downloadConfig.s3Client = m_s3Client; + downloadConfig.s3Client = m_s3Clients[GetParam()]; downloadConfig.transferStatusUpdatedCallback = [&](const TransferManager*, const std::shared_ptr& handle) { @@ -1834,26 +1878,21 @@ TEST_F(TransferTests, TransferManager_CancelAndRetryDownloadTest) } // Single part upload with compute content MD5 -TEST_F(TransferTests, TransferManager_SinglePartUploadWithComputeContentMd5Test) +TEST_P(TransferTests, TransferManager_SinglePartUploadWithComputeContentMd5Test) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String testFilePath = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(testFilePath, MB5, testString); - if (EmptyBucket(GetTestBucketName())) - { - WaitForBucketToEmpty(GetTestBucketName()); - } - GetObjectRequest getObjectRequest; getObjectRequest.SetBucket(GetTestBucketName()); getObjectRequest.SetKey(RandomFileName); - GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest); + GetObjectOutcome getObjectOutcome = m_s3Clients[GetParam()]->GetObject(getObjectRequest); EXPECT_FALSE(getObjectOutcome.IsSuccess()); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; transferManagerConfig.computeContentMD5 = true; auto transferManager = TransferManager::Create(transferManagerConfig); @@ -1870,7 +1909,7 @@ TEST_F(TransferTests, TransferManager_SinglePartUploadWithComputeContentMd5Test) headObjectRequest.SetBucket(GetTestBucketName()); headObjectRequest.SetKey(RandomFileName); - HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest); + HeadObjectOutcome headObjectOutcome = m_s3Clients[GetParam()]->HeadObject(headObjectRequest); AWS_ASSERT_SUCCESS(headObjectOutcome); VerifyUploadedFile(*transferManager, @@ -1882,22 +1921,17 @@ TEST_F(TransferTests, TransferManager_SinglePartUploadWithComputeContentMd5Test) } // Multipart upload with compute content MD5 -TEST_F(TransferTests, MultipartUploadWithComputeContentMd5Test) +TEST_P(TransferTests, MultipartUploadWithComputeContentMd5Test) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String testFilePath = MakeFilePath(RandomFileName.c_str()); ScopedTestFile testFile(testFilePath, MEDIUM_TEST_SIZE, testString); - if (EmptyBucket(GetTestBucketName())) - { - WaitForBucketToEmpty(GetTestBucketName()); - } - GetObjectRequest getObjectRequest; getObjectRequest.SetBucket(GetTestBucketName()); getObjectRequest.SetKey(RandomFileName); - GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest); + GetObjectOutcome getObjectOutcome = m_s3Clients[GetParam()]->GetObject(getObjectRequest); EXPECT_FALSE(getObjectOutcome.IsSuccess()); ListMultipartUploadsRequest listMultipartRequest; @@ -1905,7 +1939,7 @@ TEST_F(TransferTests, MultipartUploadWithComputeContentMd5Test) listMultipartRequest.SetBucket(GetTestBucketName()); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; transferManagerConfig.computeContentMD5 = true; auto transferManager = TransferManager::Create(transferManagerConfig); @@ -1931,7 +1965,7 @@ TEST_F(TransferTests, MultipartUploadWithComputeContentMd5Test) headObjectRequest.SetBucket(GetTestBucketName()); headObjectRequest.SetKey(RandomFileName); - HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest); + HeadObjectOutcome headObjectOutcome = m_s3Clients[GetParam()]->HeadObject(headObjectRequest); AWS_ASSERT_SUCCESS(headObjectOutcome); VerifyUploadedFile(*transferManager, @@ -1943,7 +1977,7 @@ TEST_F(TransferTests, MultipartUploadWithComputeContentMd5Test) } // Test of a basic multi part upload - 7.5 megs with the templated tests -TEST_F(TransferTests, TransferManager_TemplatesTest) +TEST_P(TransferTests, TransferManager_TemplatesTest) { const Aws::String RandomFileName = Aws::Utils::UUID::RandomUUID(); Aws::String mediumTestFilePath = MakeFilePath(RandomFileName.c_str()); @@ -1956,7 +1990,7 @@ TEST_F(TransferTests, TransferManager_TemplatesTest) putObjReqTemplate.WithRequestPayer(Aws::S3::Model::RequestPayer::requester); TransferManagerConfiguration transferManagerConfig(m_executor.get()); - transferManagerConfig.s3Client = m_s3Client; + transferManagerConfig.s3Client = m_s3Clients[GetParam()]; transferManagerConfig.putObjectTemplate = putObjReqTemplate; transferManagerConfig.getObjectTemplate = getObjReqTemplate; @@ -2018,7 +2052,7 @@ class TestHelperTransferManager : public Aws::Transfer::TransferManager }; }; -TEST_F(TransferTests, TransferManager_TestRelativePrefix) +TEST_P(TransferTests, TransferManager_TestRelativePrefix) { static const std::vector TEST_CASES = { @@ -2078,4 +2112,7 @@ TEST_F(TransferTests, TransferManager_TestRelativePrefix) } } +INSTANTIATE_TEST_SUITE_P(Https, TransferTests, testing::Values(TestType::Https)); +INSTANTIATE_TEST_SUITE_P(Http, TransferTests, testing::Values(TestType::Http)); + }