Add ability to set metadata for object uploads in transfer library.

aws-cpp-sdk-transfer-tests/TransferTests.cpp

@@ -1188,5 +1188,101 @@ TEST_F(TransferTests, ScopeTests)
     }
 }
 
+// Single part upload with metadata specified
+TEST_F(TransferTests, SinglePartUploadWithMetadataTest)
+{
+    if (EmptyBucket(GetTestBucketName()))
+    {
+        WaitForBucketToEmpty(GetTestBucketName());
+    }
+    GetObjectRequest getObjectRequest;
+    getObjectRequest.SetBucket(GetTestBucketName());
+    getObjectRequest.SetKey(TEST_FILE_NAME);
+
+    GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest);
+    EXPECT_FALSE(getObjectOutcome.IsSuccess());
+
+    Aws::Map<Aws::String, Aws::String> metadata;
+    metadata["key1"] = "val1";
+    metadata["key2"] = "val2";
+    const bool cCreateBucket = true;
+    const bool cConsistencyChecks = false;
+    std::shared_ptr<UploadFileRequest> requestPtr = m_transferClient->UploadFile(TEST_FILE_NAME, GetTestBucketName(), "", "", metadata, cCreateBucket, cConsistencyChecks);
+
+    ASSERT_FALSE(requestPtr->IsDone());
+
+    WaitForUploadAndUpdate(requestPtr, 100.0f);
+
+    ASSERT_TRUE(requestPtr->IsDone());
+
+    ASSERT_TRUE(requestPtr->CompletedSuccessfully());
+
+    WaitForObjectToPropagate(GetTestBucketName(), TEST_FILE_NAME);
+
+    // Check the metadata matches
+    HeadObjectRequest headObjectRequest;
+    headObjectRequest.SetBucket(GetTestBucketName());
+    headObjectRequest.SetKey(TEST_FILE_NAME);
+
+    HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest);
+    ASSERT_TRUE(headObjectOutcome.IsSuccess());
+
+    Aws::Map<Aws::String, Aws::String> headObjectMetadata = headObjectOutcome.GetResult().GetMetadata();
+    ASSERT_EQ(metadata.size(), headObjectMetadata.size());
+    ASSERT_EQ(metadata["key1"], headObjectMetadata["key1"]);
+    ASSERT_EQ(metadata["key2"], headObjectMetadata["key2"]);
+
+}
+
+// Multipart upload with metadata specified
+TEST_F(TransferTests, MultipartUploadWithMetadataTest)
+{
+    if (EmptyBucket(GetTestBucketName()))
+    {
+        WaitForBucketToEmpty(GetTestBucketName());
+    }
+
+    GetObjectRequest getObjectRequest;
+    getObjectRequest.SetBucket(GetTestBucketName());
+    getObjectRequest.SetKey(MEDIUM_FILE_KEY);
+
+    GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest);
+    EXPECT_FALSE(getObjectOutcome.IsSuccess());
+
+    ListMultipartUploadsRequest listMultipartRequest;
+
+    listMultipartRequest.SetBucket(GetTestBucketName());
+
+    Aws::Map<Aws::String, Aws::String> metadata;
+    metadata["key1"] = "val1";
+    metadata["key2"] = "val2";
+    const bool cCreateBucket = true;
+    const bool cConsistencyChecks = false;
+    std::shared_ptr<UploadFileRequest> requestPtr = m_transferClient->UploadFile(MEDIUM_TEST_FILE_NAME, GetTestBucketName(), MEDIUM_FILE_KEY, "", metadata, cCreateBucket, cConsistencyChecks);
+
+    ASSERT_FALSE(requestPtr->IsDone());
+
+    WaitForUploadAndUpdate(requestPtr, 100.0f);
+
+    ASSERT_TRUE(requestPtr->IsDone());
+    ASSERT_TRUE(requestPtr->CompletedSuccessfully());
+
+    WaitForObjectToPropagate(GetTestBucketName(), MEDIUM_FILE_KEY);
+
+    // Check the metadata matches
+    HeadObjectRequest headObjectRequest;
+    headObjectRequest.SetBucket(GetTestBucketName());
+    headObjectRequest.SetKey(MEDIUM_FILE_KEY);
+
+    HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest);
+    ASSERT_TRUE(headObjectOutcome.IsSuccess());
+
+    Aws::Map<Aws::String, Aws::String> headObjectMetadata = headObjectOutcome.GetResult().GetMetadata();
+    ASSERT_EQ(metadata.size(), headObjectMetadata.size());
+    ASSERT_EQ(metadata["key1"], headObjectMetadata["key1"]);
+    ASSERT_EQ(metadata["key2"], headObjectMetadata["key2"]);
+
+}
+
 
 }

aws-cpp-sdk-transfer/include/aws/transfer/TransferClient.h

@@ -61,10 +61,13 @@ class AWS_TRANSFER_API TransferClient
         TransferClient(const std::shared_ptr<Aws::S3::S3Client>& s3Client, const TransferClientConfiguration& config);
         ~TransferClient();
 
-        // Single entry point for attempting an upload - attempting to create an existing bucket won't hurt anything but will affect performance
+        // Entry point for attempting an upload - attempting to create an existing bucket won't hurt anything but will affect performance
         // unnecessarily as the request waits for S3 to propagate the bucket
         // All queries about the upload after this point can be found in UploadFileRequest's interface
         std::shared_ptr<UploadFileRequest> UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, bool createBucket = false, bool doConsistencyChecks = false);
+        // Entry point similar to above but with metadata specified
+        std::shared_ptr<UploadFileRequest> UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, const Aws::Map<Aws::String, Aws::String>& metadata, bool createBucket = false, bool doConsistencyChecks = false);
+        std::shared_ptr<UploadFileRequest> UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, Aws::Map<Aws::String, Aws::String>&& metadata, bool createBucket = false, bool doConsistencyChecks = false);
 
         // User requested upload cancels should go through here
         void CancelUpload(std::shared_ptr<UploadFileRequest>& fileRequest) const;
@@ -84,7 +87,7 @@ class AWS_TRANSFER_API TransferClient
     private:
 
         void UploadFileInternal(std::shared_ptr<UploadFileRequest>& fileRequest);
-  
+
         void ProcessSingleBuffer(std::shared_ptr<UploadFileRequest>& request, const std::shared_ptr<UploadBuffer>& buffer);
 
         void CancelUploadInternal(std::shared_ptr<UploadFileRequest>& fileRequest) const;
@@ -168,4 +171,4 @@ class AWS_TRANSFER_API TransferClient
 } // namespace Transfer
 } // namespace Aws
 
-#endif
+#endif

aws-cpp-sdk-transfer/include/aws/transfer/UploadFileRequest.h

@@ -76,14 +76,30 @@ struct AWS_TRANSFER_API PartRequestRecord
 class AWS_TRANSFER_API UploadFileRequest : public S3FileRequest, public std::enable_shared_from_this<UploadFileRequest>
 {
 public:
+    UploadFileRequest(const Aws::String& fileName,
+                      const Aws::String& bucketName,
+                      const Aws::String& keyName,
+                      const Aws::String& contentType,
+                      Aws::Map<Aws::String, Aws::String>&& metadata,
+                      const std::shared_ptr<Aws::S3::S3Client>& s3Client,
+                      bool createBucket,
+                      bool doConsistencyChecks);
     UploadFileRequest(const Aws::String& fileName, 
                       const Aws::String& bucketName, 
                       const Aws::String& keyName, 
                       const Aws::String& contentType, 
                       const std::shared_ptr<Aws::S3::S3Client>& s3Client, 
                       bool createBucket,
                       bool doConsistencyChecks);
-    ~UploadFileRequest();
+    UploadFileRequest(const Aws::String& fileName,
+                      const Aws::String& bucketName,
+                      const Aws::String& keyName,
+                      const Aws::String& contentType,
+                      const Aws::Map<Aws::String, Aws::String>& metadata,
+                      const std::shared_ptr<Aws::S3::S3Client>& s3Client,
+                      bool createBucket,
+                      bool doConsistencyChecks);
+    virtual ~UploadFileRequest();
 
     // How many parts have we at least begun to upload
     uint32_t GetPartCount() const;
@@ -255,6 +271,7 @@ class AWS_TRANSFER_API UploadFileRequest : public S3FileRequest, public std::ena
     Aws::List<std::shared_ptr<UploadBuffer> > m_buffersReady;
 
     Aws::String m_contentType;
+    Aws::Map<Aws::String, Aws::String> m_metadata;
     Aws::String m_uploadId;
     uint32_t m_createMultipartRetries;
     uint32_t m_createBucketRetries;

aws-cpp-sdk-transfer/source/transfer/TransferClient.cpp

@@ -74,6 +74,24 @@ std::shared_ptr<UploadFileRequest> TransferClient::UploadFile(const Aws::String&
     return request;
 }
 
+std::shared_ptr<UploadFileRequest> TransferClient::UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, const Aws::Map<Aws::String, Aws::String>& metadata, bool createBucket, bool doConsistencyChecks)
+{
+    auto request = Aws::MakeShared<UploadFileRequest>(ALLOCATION_TAG, fileName, bucketName, keyName, contentType, metadata, m_s3Client, createBucket, doConsistencyChecks);
+
+    UploadFileInternal(request);
+
+    return request;
+}
+
+std::shared_ptr<UploadFileRequest> TransferClient::UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, Aws::Map<Aws::String, Aws::String>&& metadata, bool createBucket, bool doConsistencyChecks)
+{
+    auto request = Aws::MakeShared<UploadFileRequest>(ALLOCATION_TAG, fileName, bucketName, keyName, contentType, std::move(metadata), m_s3Client, createBucket, doConsistencyChecks);
+
+    UploadFileInternal(request);
+
+    return request;
+}
+
 void TransferClient::UploadFileInternal(std::shared_ptr<UploadFileRequest>& request) 
 {
 

aws-cpp-sdk-transfer/source/transfer/UploadFileRequest.cpp

@@ -55,6 +55,7 @@ UploadFileRequest::UploadFileRequest(const Aws::String& fileName,
                                      const Aws::String& bucketName, 
                                      const Aws::String& keyName, 
                                      const Aws::String& contentType, 
+                                     Aws::Map<Aws::String, Aws::String>&& metadata,
                                      const std::shared_ptr<Aws::S3::S3Client>& s3Client, 
                                      bool createBucket,
                                      bool doConsistencyChecks) :
@@ -73,6 +74,7 @@ m_bucketPropagated(false),
 m_totalParts(0),
 m_fileStream(fileName.c_str(), std::ios::binary | std::ios::ate),
 m_contentType(contentType),
+m_metadata(std::move(metadata)),
 m_createMultipartRetries(0),
 m_createBucketRetries(0),
 m_completeRetries(0),
@@ -105,6 +107,29 @@ m_headBucketRetries(0)
     }
 }
 
+UploadFileRequest::UploadFileRequest(const Aws::String& fileName,
+                                     const Aws::String& bucketName,
+                                     const Aws::String& keyName,
+                                     const Aws::String& contentType,
+                                     const std::shared_ptr<Aws::S3::S3Client>& s3Client,
+                                     bool createBucket,
+                                     bool doConsistencyChecks) :
+UploadFileRequest(fileName, bucketName, keyName, contentType, {}, s3Client, createBucket, doConsistencyChecks)
+{
+}
+
+UploadFileRequest::UploadFileRequest(const Aws::String& fileName,
+                                     const Aws::String& bucketName,
+                                     const Aws::String& keyName,
+                                     const Aws::String& contentType,
+                                     const Aws::Map<Aws::String, Aws::String>& metadata,
+                                     const std::shared_ptr<Aws::S3::S3Client>& s3Client,
+                                     bool createBucket,
+                                     bool doConsistencyChecks) :
+UploadFileRequest(fileName, bucketName, keyName, contentType, Aws::Map<Aws::String, Aws::String>(metadata), s3Client, createBucket, doConsistencyChecks)
+{
+}
+
 UploadFileRequest::~UploadFileRequest()
 {
     m_fileStream.close();
@@ -202,6 +227,10 @@ bool UploadFileRequest::CreateMultipartUpload()
     {
         createMultipartUploadRequest.SetContentType(m_contentType);
     }
+    if (m_metadata.size() > 0)
+    {
+        createMultipartUploadRequest.SetMetadata(m_metadata);
+    }
 
     std::shared_ptr<Aws::Client::AsyncCallerContext> context = Aws::MakeShared<UploadFileContext>(ALLOCATION_TAG, shared_from_this());
 
@@ -507,7 +536,7 @@ bool UploadFileRequest::ProcessBuffer(const std::shared_ptr<UploadBuffer>& buffe
         return false;
     }
 
-    if (GetTotalParts() == 1)
+    if (IsSinglePartUpload())
     {
         // Don't need more than one part, do everything now
         return DoSingleObjectUpload(streamBuf, bytesRead);
@@ -774,6 +803,10 @@ bool UploadFileRequest::DoSingleObjectUpload(std::shared_ptr<Aws::IOStream>& str
     {
         putObjectRequest.SetContentType(m_contentType);
     }
+    if (m_metadata.size() > 0)
+    {
+        putObjectRequest.SetMetadata(m_metadata);
+    }
     putObjectRequest.SetKey(GetKeyName());
 
     putObjectRequest.SetDataSentEventHandler(std::bind(&UploadFileRequest::OnDataSent, this, std::placeholders::_1, std::placeholders::_2));