Skip to content

Commit 50ff929

Browse files
kahkengkahkeng
committed
Add ability to set metadata for object uploads in transfer library.
1 parent eab46cc commit 50ff929

File tree

5 files changed

+172
-5
lines changed

5 files changed

+172
-5
lines changed

‎aws-cpp-sdk-transfer-tests/TransferTests.cpp

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1188,5 +1188,101 @@ TEST_F(TransferTests, ScopeTests)
11881188
}
11891189
}
11901190

1191+
// Single part upload with metadata specified
1192+
TEST_F(TransferTests, SinglePartUploadWithMetadataTest)
1193+
{
1194+
if (EmptyBucket(GetTestBucketName()))
1195+
{
1196+
WaitForBucketToEmpty(GetTestBucketName());
1197+
}
1198+
GetObjectRequest getObjectRequest;
1199+
getObjectRequest.SetBucket(GetTestBucketName());
1200+
getObjectRequest.SetKey(TEST_FILE_NAME);
1201+
1202+
GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest);
1203+
EXPECT_FALSE(getObjectOutcome.IsSuccess());
1204+
1205+
Aws::Map<Aws::String, Aws::String> metadata;
1206+
metadata["key1"] = "val1";
1207+
metadata["key2"] = "val2";
1208+
const bool cCreateBucket = false;
1209+
const bool cConsistencyChecks = false;
1210+
std::shared_ptr<UploadFileRequest> requestPtr = m_transferClient->UploadFile(TEST_FILE_NAME, GetTestBucketName(), "", "", metadata, cCreateBucket, cConsistencyChecks);
1211+
1212+
ASSERT_FALSE(requestPtr->IsDone());
1213+
1214+
WaitForUploadAndUpdate(requestPtr, 100.0f);
1215+
1216+
ASSERT_TRUE(requestPtr->IsDone());
1217+
1218+
ASSERT_TRUE(requestPtr->CompletedSuccessfully());
1219+
1220+
WaitForObjectToPropagate(GetTestBucketName(), TEST_FILE_NAME);
1221+
1222+
// Check the metadata matches
1223+
HeadObjectRequest headObjectRequest;
1224+
headObjectRequest.SetBucket(GetTestBucketName());
1225+
headObjectRequest.SetKey(TEST_FILE_NAME);
1226+
1227+
HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest);
1228+
ASSERT_TRUE(headObjectOutcome.IsSuccess());
1229+
1230+
Aws::Map<Aws::String, Aws::String> headObjectMetadata = headObjectOutcome.GetResult().GetMetadata();
1231+
ASSERT_EQ(metadata.size(), headObjectMetadata.size());
1232+
ASSERT_EQ(metadata["key1"], headObjectMetadata["key1"]);
1233+
ASSERT_EQ(metadata["key2"], headObjectMetadata["key2"]);
1234+
1235+
}
1236+
1237+
// Multipart upload with metadata specified
1238+
TEST_F(TransferTests, MultipartUploadWithMetadataTest)
1239+
{
1240+
if (EmptyBucket(GetTestBucketName()))
1241+
{
1242+
WaitForBucketToEmpty(GetTestBucketName());
1243+
}
1244+
1245+
GetObjectRequest getObjectRequest;
1246+
getObjectRequest.SetBucket(GetTestBucketName());
1247+
getObjectRequest.SetKey(MEDIUM_FILE_KEY);
1248+
1249+
GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest);
1250+
EXPECT_FALSE(getObjectOutcome.IsSuccess());
1251+
1252+
ListMultipartUploadsRequest listMultipartRequest;
1253+
1254+
listMultipartRequest.SetBucket(GetTestBucketName());
1255+
1256+
Aws::Map<Aws::String, Aws::String> metadata;
1257+
metadata["key1"] = "val1";
1258+
metadata["key2"] = "val2";
1259+
const bool cCreateBucket = false;
1260+
const bool cConsistencyChecks = false;
1261+
std::shared_ptr<UploadFileRequest> requestPtr = m_transferClient->UploadFile(MEDIUM_TEST_FILE_NAME, GetTestBucketName(), MEDIUM_FILE_KEY, "", metadata, cCreateBucket, cConsistencyChecks);
1262+
1263+
ASSERT_FALSE(requestPtr->IsDone());
1264+
1265+
WaitForUploadAndUpdate(requestPtr, 100.0f);
1266+
1267+
ASSERT_TRUE(requestPtr->IsDone());
1268+
ASSERT_TRUE(requestPtr->CompletedSuccessfully());
1269+
1270+
WaitForObjectToPropagate(GetTestBucketName(), MEDIUM_FILE_KEY);
1271+
1272+
// Check the metadata matches
1273+
HeadObjectRequest headObjectRequest;
1274+
headObjectRequest.SetBucket(GetTestBucketName());
1275+
headObjectRequest.SetKey(MEDIUM_FILE_KEY);
1276+
1277+
HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest);
1278+
ASSERT_TRUE(headObjectOutcome.IsSuccess());
1279+
1280+
Aws::Map<Aws::String, Aws::String> headObjectMetadata = headObjectOutcome.GetResult().GetMetadata();
1281+
ASSERT_EQ(metadata.size(), headObjectMetadata.size());
1282+
ASSERT_EQ(metadata["key1"], headObjectMetadata["key1"]);
1283+
ASSERT_EQ(metadata["key2"], headObjectMetadata["key2"]);
1284+
1285+
}
1286+
11911287

11921288
}

‎aws-cpp-sdk-transfer/include/aws/transfer/TransferClient.h

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,13 @@ class AWS_TRANSFER_API TransferClient
6161
TransferClient(const std::shared_ptr<Aws::S3::S3Client>& s3Client, const TransferClientConfiguration& config);
6262
~TransferClient();
6363

64-
// Single entry point for attempting an upload - attempting to create an existing bucket won't hurt anything but will affect performance
64+
// Entry point for attempting an upload - attempting to create an existing bucket won't hurt anything but will affect performance
6565
// unnecessarily as the request waits for S3 to propagate the bucket
6666
// All queries about the upload after this point can be found in UploadFileRequest's interface
6767
std::shared_ptr<UploadFileRequest> UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, bool createBucket = false, bool doConsistencyChecks = false);
68+
// Entry point similar to above but with metadata specified
69+
std::shared_ptr<UploadFileRequest> UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, const Aws::Map<Aws::String, Aws::String>& metadata, bool createBucket = false, bool doConsistencyChecks = false);
70+
std::shared_ptr<UploadFileRequest> UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, Aws::Map<Aws::String, Aws::String>&& metadata, bool createBucket = false, bool doConsistencyChecks = false);
6871

6972
// User requested upload cancels should go through here
7073
void CancelUpload(std::shared_ptr<UploadFileRequest>& fileRequest) const;
@@ -84,7 +87,7 @@ class AWS_TRANSFER_API TransferClient
8487
private:
8588

8689
void UploadFileInternal(std::shared_ptr<UploadFileRequest>& fileRequest);
87-
90+
8891
void ProcessSingleBuffer(std::shared_ptr<UploadFileRequest>& request, const std::shared_ptr<UploadBuffer>& buffer);
8992

9093
void CancelUploadInternal(std::shared_ptr<UploadFileRequest>& fileRequest) const;
@@ -168,4 +171,4 @@ class AWS_TRANSFER_API TransferClient
168171
} // namespace Transfer
169172
} // namespace Aws
170173

171-
#endif
174+
#endif

‎aws-cpp-sdk-transfer/include/aws/transfer/UploadFileRequest.h

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,14 +76,30 @@ struct AWS_TRANSFER_API PartRequestRecord
7676
class AWS_TRANSFER_API UploadFileRequest : public S3FileRequest, public std::enable_shared_from_this<UploadFileRequest>
7777
{
7878
public:
79+
UploadFileRequest(const Aws::String& fileName,
80+
const Aws::String& bucketName,
81+
const Aws::String& keyName,
82+
const Aws::String& contentType,
83+
Aws::Map<Aws::String, Aws::String>&& metadata,
84+
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
85+
bool createBucket,
86+
bool doConsistencyChecks);
7987
UploadFileRequest(const Aws::String& fileName,
8088
const Aws::String& bucketName,
8189
const Aws::String& keyName,
8290
const Aws::String& contentType,
8391
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
8492
bool createBucket,
8593
bool doConsistencyChecks);
86-
~UploadFileRequest();
94+
UploadFileRequest(const Aws::String& fileName,
95+
const Aws::String& bucketName,
96+
const Aws::String& keyName,
97+
const Aws::String& contentType,
98+
const Aws::Map<Aws::String, Aws::String>& metadata,
99+
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
100+
bool createBucket,
101+
bool doConsistencyChecks);
102+
virtual ~UploadFileRequest();
87103

88104
// How many parts have we at least begun to upload
89105
uint32_t GetPartCount() const;
@@ -255,6 +271,7 @@ class AWS_TRANSFER_API UploadFileRequest : public S3FileRequest, public std::ena
255271
Aws::List<std::shared_ptr<UploadBuffer> > m_buffersReady;
256272

257273
Aws::String m_contentType;
274+
Aws::Map<Aws::String, Aws::String> m_metadata;
258275
Aws::String m_uploadId;
259276
uint32_t m_createMultipartRetries;
260277
uint32_t m_createBucketRetries;

‎aws-cpp-sdk-transfer/source/transfer/TransferClient.cpp

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,24 @@ std::shared_ptr<UploadFileRequest> TransferClient::UploadFile(const Aws::String&
7474
return request;
7575
}
7676

77+
std::shared_ptr<UploadFileRequest> TransferClient::UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, const Aws::Map<Aws::String, Aws::String>& metadata, bool createBucket, bool doConsistencyChecks)
78+
{
79+
auto request = Aws::MakeShared<UploadFileRequest>(ALLOCATION_TAG, fileName, bucketName, keyName, contentType, metadata, m_s3Client, createBucket, doConsistencyChecks);
80+
81+
UploadFileInternal(request);
82+
83+
return request;
84+
}
85+
86+
std::shared_ptr<UploadFileRequest> TransferClient::UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, Aws::Map<Aws::String, Aws::String>&& metadata, bool createBucket, bool doConsistencyChecks)
87+
{
88+
auto request = Aws::MakeShared<UploadFileRequest>(ALLOCATION_TAG, fileName, bucketName, keyName, contentType, metadata, m_s3Client, createBucket, doConsistencyChecks);
89+
90+
UploadFileInternal(request);
91+
92+
return request;
93+
}
94+
7795
void TransferClient::UploadFileInternal(std::shared_ptr<UploadFileRequest>& request)
7896
{
7997

‎aws-cpp-sdk-transfer/source/transfer/UploadFileRequest.cpp

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ UploadFileRequest::UploadFileRequest(const Aws::String& fileName,
5555
const Aws::String& bucketName,
5656
const Aws::String& keyName,
5757
const Aws::String& contentType,
58+
Aws::Map<Aws::String, Aws::String>&& metadata,
5859
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
5960
bool createBucket,
6061
bool doConsistencyChecks) :
@@ -73,6 +74,7 @@ m_bucketPropagated(false),
7374
m_totalParts(0),
7475
m_fileStream(fileName.c_str(), std::ios::binary | std::ios::ate),
7576
m_contentType(contentType),
77+
m_metadata(std::forward<Aws::Map<Aws::String, Aws::String>>(metadata)),
7678
m_createMultipartRetries(0),
7779
m_createBucketRetries(0),
7880
m_completeRetries(0),
@@ -105,6 +107,29 @@ m_headBucketRetries(0)
105107
}
106108
}
107109

110+
UploadFileRequest::UploadFileRequest(const Aws::String& fileName,
111+
const Aws::String& bucketName,
112+
const Aws::String& keyName,
113+
const Aws::String& contentType,
114+
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
115+
bool createBucket,
116+
bool doConsistencyChecks) :
117+
UploadFileRequest(fileName, bucketName, keyName, contentType, {}, s3Client, createBucket, doConsistencyChecks)
118+
{
119+
}
120+
121+
UploadFileRequest::UploadFileRequest(const Aws::String& fileName,
122+
const Aws::String& bucketName,
123+
const Aws::String& keyName,
124+
const Aws::String& contentType,
125+
const Aws::Map<Aws::String, Aws::String>& metadata,
126+
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
127+
bool createBucket,
128+
bool doConsistencyChecks) :
129+
UploadFileRequest(fileName, bucketName, keyName, contentType, Aws::Map<Aws::String, Aws::String>(metadata), s3Client, createBucket, doConsistencyChecks)
130+
{
131+
}
132+
108133
UploadFileRequest::~UploadFileRequest()
109134
{
110135
m_fileStream.close();
@@ -202,6 +227,10 @@ bool UploadFileRequest::CreateMultipartUpload()
202227
{
203228
createMultipartUploadRequest.SetContentType(m_contentType);
204229
}
230+
if (m_metadata.size() > 0)
231+
{
232+
createMultipartUploadRequest.SetMetadata(m_metadata);
233+
}
205234

206235
std::shared_ptr<Aws::Client::AsyncCallerContext> context = Aws::MakeShared<UploadFileContext>(ALLOCATION_TAG, shared_from_this());
207236

@@ -507,7 +536,7 @@ bool UploadFileRequest::ProcessBuffer(const std::shared_ptr<UploadBuffer>& buffe
507536
return false;
508537
}
509538

510-
if (GetTotalParts() == 1)
539+
if (IsSinglePartUpload())
511540
{
512541
// Don't need more than one part, do everything now
513542
return DoSingleObjectUpload(streamBuf, bytesRead);
@@ -774,6 +803,10 @@ bool UploadFileRequest::DoSingleObjectUpload(std::shared_ptr<Aws::IOStream>& str
774803
{
775804
putObjectRequest.SetContentType(m_contentType);
776805
}
806+
if (m_metadata.size() > 0)
807+
{
808+
putObjectRequest.SetMetadata(m_metadata);
809+
}
777810
putObjectRequest.SetKey(GetKeyName());
778811

779812
putObjectRequest.SetDataSentEventHandler(std::bind(&UploadFileRequest::OnDataSent, this, std::placeholders::_1, std::placeholders::_2));

0 commit comments

Comments
 (0)