Skip to content

Commit 766c6ed

Browse files
kahkengkahkeng
committed
Add ability to set metadata for object uploads in transfer library.
1 parent c3ccf3e commit 766c6ed

File tree

5 files changed

+176
-4
lines changed

5 files changed

+176
-4
lines changed

‎aws-cpp-sdk-transfer-tests/TransferTests.cpp

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1188,5 +1188,101 @@ TEST_F(TransferTests, ScopeTests)
11881188
}
11891189
}
11901190

1191+
// Single part upload with metadata specified
1192+
TEST_F(TransferTests, SinglePartUploadWithMetadataTest)
1193+
{
1194+
if (EmptyBucket(GetTestBucketName()))
1195+
{
1196+
WaitForBucketToEmpty(GetTestBucketName());
1197+
}
1198+
GetObjectRequest getObjectRequest;
1199+
getObjectRequest.SetBucket(GetTestBucketName());
1200+
getObjectRequest.SetKey(TEST_FILE_NAME);
1201+
1202+
GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest);
1203+
EXPECT_FALSE(getObjectOutcome.IsSuccess());
1204+
1205+
Aws::Map<Aws::String, Aws::String> metadata;
1206+
metadata["key1"] = "val1";
1207+
metadata["key2"] = "val2";
1208+
const bool cCreateBucket = false;
1209+
const bool cConsistencyChecks = false;
1210+
std::shared_ptr<UploadFileRequest> requestPtr = m_transferClient->UploadFile(TEST_FILE_NAME, GetTestBucketName(), "", "", metadata, cCreateBucket, cConsistencyChecks);
1211+
1212+
ASSERT_FALSE(requestPtr->IsDone());
1213+
1214+
WaitForUploadAndUpdate(requestPtr, 100.0f);
1215+
1216+
ASSERT_TRUE(requestPtr->IsDone());
1217+
1218+
ASSERT_TRUE(requestPtr->CompletedSuccessfully());
1219+
1220+
WaitForObjectToPropagate(GetTestBucketName(), TEST_FILE_NAME);
1221+
1222+
// Check the metadata matches
1223+
HeadObjectRequest headObjectRequest;
1224+
headObjectRequest.SetBucket(GetTestBucketName());
1225+
headObjectRequest.SetKey(TEST_FILE_NAME);
1226+
1227+
HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest);
1228+
ASSERT_TRUE(headObjectOutcome.IsSuccess());
1229+
1230+
Aws::Map<Aws::String, Aws::String> headObjectMetadata = headObjectOutcome.GetResult().GetMetadata();
1231+
ASSERT_EQ(2u, headObjectMetadata.size());
1232+
ASSERT_EQ("val1", headObjectMetadata["key1"]);
1233+
ASSERT_EQ("val2", headObjectMetadata["key2"]);
1234+
1235+
}
1236+
1237+
// Multipart upload with metadata specified
1238+
TEST_F(TransferTests, MultipartUploadWithMetadataTest)
1239+
{
1240+
if (EmptyBucket(GetTestBucketName()))
1241+
{
1242+
WaitForBucketToEmpty(GetTestBucketName());
1243+
}
1244+
1245+
GetObjectRequest getObjectRequest;
1246+
getObjectRequest.SetBucket(GetTestBucketName());
1247+
getObjectRequest.SetKey(MEDIUM_FILE_KEY);
1248+
1249+
GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest);
1250+
EXPECT_FALSE(getObjectOutcome.IsSuccess());
1251+
1252+
ListMultipartUploadsRequest listMultipartRequest;
1253+
1254+
listMultipartRequest.SetBucket(GetTestBucketName());
1255+
1256+
Aws::Map<Aws::String, Aws::String> metadata;
1257+
metadata["key1"] = "val1";
1258+
metadata["key2"] = "val2";
1259+
const bool cCreateBucket = false;
1260+
const bool cConsistencyChecks = false;
1261+
std::shared_ptr<UploadFileRequest> requestPtr = m_transferClient->UploadFile(MEDIUM_TEST_FILE_NAME, GetTestBucketName(), MEDIUM_FILE_KEY, "", metadata, cCreateBucket, cConsistencyChecks);
1262+
1263+
ASSERT_FALSE(requestPtr->IsDone());
1264+
1265+
WaitForUploadAndUpdate(requestPtr, 100.0f);
1266+
1267+
ASSERT_TRUE(requestPtr->IsDone());
1268+
ASSERT_TRUE(requestPtr->CompletedSuccessfully());
1269+
1270+
WaitForObjectToPropagate(GetTestBucketName(), MEDIUM_FILE_KEY);
1271+
1272+
// Check the metadata matches
1273+
HeadObjectRequest headObjectRequest;
1274+
headObjectRequest.SetBucket(GetTestBucketName());
1275+
headObjectRequest.SetKey(MEDIUM_FILE_KEY);
1276+
1277+
HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest);
1278+
ASSERT_TRUE(headObjectOutcome.IsSuccess());
1279+
1280+
Aws::Map<Aws::String, Aws::String> headObjectMetadata = headObjectOutcome.GetResult().GetMetadata();
1281+
ASSERT_EQ(2u, headObjectMetadata.size());
1282+
ASSERT_EQ("val1", headObjectMetadata["key1"]);
1283+
ASSERT_EQ("val2", headObjectMetadata["key2"]);
1284+
1285+
}
1286+
11911287

11921288
}

‎aws-cpp-sdk-transfer/include/aws/transfer/TransferClient.h

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,13 @@ class AWS_TRANSFER_API TransferClient
6161
TransferClient(const std::shared_ptr<Aws::S3::S3Client>& s3Client, const TransferClientConfiguration& config);
6262
~TransferClient();
6363

64-
// Single entry point for attempting an upload - attempting to create an existing bucket won't hurt anything but will affect performance
64+
// Entry point for attempting an upload - attempting to create an existing bucket won't hurt anything but will affect performance
6565
// unnecessarily as the request waits for S3 to propagate the bucket
6666
// All queries about the upload after this point can be found in UploadFileRequest's interface
6767
std::shared_ptr<UploadFileRequest> UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, bool createBucket = false, bool doConsistencyChecks = false);
68+
// Entry point similar to above but with metadata specified
69+
std::shared_ptr<UploadFileRequest> UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, const Aws::Map<Aws::String, Aws::String>& metadata, bool createBucket = false, bool doConsistencyChecks = false);
70+
std::shared_ptr<UploadFileRequest> UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, Aws::Map<Aws::String, Aws::String>&& metadata, bool createBucket = false, bool doConsistencyChecks = false);
6871

6972
// User requested upload cancels should go through here
7073
void CancelUpload(std::shared_ptr<UploadFileRequest>& fileRequest) const;
@@ -84,7 +87,7 @@ class AWS_TRANSFER_API TransferClient
8487
private:
8588

8689
void UploadFileInternal(std::shared_ptr<UploadFileRequest>& fileRequest);
87-
90+
8891
void ProcessSingleBuffer(std::shared_ptr<UploadFileRequest>& request, const std::shared_ptr<UploadBuffer>& buffer);
8992

9093
void CancelUploadInternal(std::shared_ptr<UploadFileRequest>& fileRequest) const;
@@ -168,4 +171,4 @@ class AWS_TRANSFER_API TransferClient
168171
} // namespace Transfer
169172
} // namespace Aws
170173

171-
#endif
174+
#endif

‎aws-cpp-sdk-transfer/include/aws/transfer/UploadFileRequest.h

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,22 @@ class AWS_TRANSFER_API UploadFileRequest : public S3FileRequest, public std::ena
8383
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
8484
bool createBucket,
8585
bool doConsistencyChecks);
86+
UploadFileRequest(const Aws::String& fileName,
87+
const Aws::String& bucketName,
88+
const Aws::String& keyName,
89+
const Aws::String& contentType,
90+
const Aws::Map<Aws::String, Aws::String>& metadata,
91+
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
92+
bool createBucket,
93+
bool doConsistencyChecks);
94+
UploadFileRequest(const Aws::String& fileName,
95+
const Aws::String& bucketName,
96+
const Aws::String& keyName,
97+
const Aws::String& contentType,
98+
Aws::Map<Aws::String, Aws::String>&& metadata,
99+
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
100+
bool createBucket,
101+
bool doConsistencyChecks);
86102
~UploadFileRequest();
87103

88104
// How many parts have we at least begun to upload
@@ -255,6 +271,8 @@ class AWS_TRANSFER_API UploadFileRequest : public S3FileRequest, public std::ena
255271
Aws::List<std::shared_ptr<UploadBuffer> > m_buffersReady;
256272

257273
Aws::String m_contentType;
274+
Aws::Map<Aws::String, Aws::String> m_metadata;
275+
bool m_metadataHasBeenSet;
258276
Aws::String m_uploadId;
259277
uint32_t m_createMultipartRetries;
260278
uint32_t m_createBucketRetries;

‎aws-cpp-sdk-transfer/source/transfer/TransferClient.cpp

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,24 @@ std::shared_ptr<UploadFileRequest> TransferClient::UploadFile(const Aws::String&
7474
return request;
7575
}
7676

77+
std::shared_ptr<UploadFileRequest> TransferClient::UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, const Aws::Map<Aws::String, Aws::String>& metadata, bool createBucket, bool doConsistencyChecks)
78+
{
79+
auto request = Aws::MakeShared<UploadFileRequest>(ALLOCATION_TAG, fileName, bucketName, keyName, contentType, metadata, m_s3Client, createBucket, doConsistencyChecks);
80+
81+
UploadFileInternal(request);
82+
83+
return request;
84+
}
85+
86+
std::shared_ptr<UploadFileRequest> TransferClient::UploadFile(const Aws::String& fileName, const Aws::String& bucketName, const Aws::String& keyName, const Aws::String& contentType, Aws::Map<Aws::String, Aws::String>&& metadata, bool createBucket, bool doConsistencyChecks)
87+
{
88+
auto request = Aws::MakeShared<UploadFileRequest>(ALLOCATION_TAG, fileName, bucketName, keyName, contentType, metadata, m_s3Client, createBucket, doConsistencyChecks);
89+
90+
UploadFileInternal(request);
91+
92+
return request;
93+
}
94+
7795
void TransferClient::UploadFileInternal(std::shared_ptr<UploadFileRequest>& request)
7896
{
7997

‎aws-cpp-sdk-transfer/source/transfer/UploadFileRequest.cpp

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ m_bucketPropagated(false),
7373
m_totalParts(0),
7474
m_fileStream(fileName.c_str(), std::ios::binary | std::ios::ate),
7575
m_contentType(contentType),
76+
m_metadataHasBeenSet(false),
7677
m_createMultipartRetries(0),
7778
m_createBucketRetries(0),
7879
m_completeRetries(0),
@@ -105,6 +106,34 @@ m_headBucketRetries(0)
105106
}
106107
}
107108

109+
UploadFileRequest::UploadFileRequest(const Aws::String& fileName,
110+
const Aws::String& bucketName,
111+
const Aws::String& keyName,
112+
const Aws::String& contentType,
113+
const Aws::Map<Aws::String, Aws::String>& metadata,
114+
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
115+
bool createBucket,
116+
bool doConsistencyChecks) :
117+
UploadFileRequest(fileName, bucketName, keyName, contentType, s3Client, createBucket, doConsistencyChecks)
118+
{
119+
m_metadataHasBeenSet = true;
120+
m_metadata = metadata;
121+
}
122+
123+
UploadFileRequest::UploadFileRequest(const Aws::String& fileName,
124+
const Aws::String& bucketName,
125+
const Aws::String& keyName,
126+
const Aws::String& contentType,
127+
Aws::Map<Aws::String, Aws::String>&& metadata,
128+
const std::shared_ptr<Aws::S3::S3Client>& s3Client,
129+
bool createBucket,
130+
bool doConsistencyChecks) :
131+
UploadFileRequest(fileName, bucketName, keyName, contentType, s3Client, createBucket, doConsistencyChecks)
132+
{
133+
m_metadataHasBeenSet = true;
134+
m_metadata = metadata;
135+
}
136+
108137
UploadFileRequest::~UploadFileRequest()
109138
{
110139
m_fileStream.close();
@@ -202,6 +231,10 @@ bool UploadFileRequest::CreateMultipartUpload()
202231
{
203232
createMultipartUploadRequest.SetContentType(m_contentType);
204233
}
234+
if (m_metadataHasBeenSet)
235+
{
236+
createMultipartUploadRequest.SetMetadata(m_metadata);
237+
}
205238

206239
std::shared_ptr<Aws::Client::AsyncCallerContext> context = Aws::MakeShared<UploadFileContext>(ALLOCATION_TAG, shared_from_this());
207240

@@ -507,7 +540,7 @@ bool UploadFileRequest::ProcessBuffer(const std::shared_ptr<UploadBuffer>& buffe
507540
return false;
508541
}
509542

510-
if (GetTotalParts() == 1)
543+
if (IsSinglePartUpload())
511544
{
512545
// Don't need more than one part, do everything now
513546
return DoSingleObjectUpload(streamBuf, bytesRead);
@@ -774,6 +807,10 @@ bool UploadFileRequest::DoSingleObjectUpload(std::shared_ptr<Aws::IOStream>& str
774807
{
775808
putObjectRequest.SetContentType(m_contentType);
776809
}
810+
if (m_metadataHasBeenSet)
811+
{
812+
putObjectRequest.SetMetadata(m_metadata);
813+
}
777814
putObjectRequest.SetKey(GetKeyName());
778815

779816
putObjectRequest.SetDataSentEventHandler(std::bind(&UploadFileRequest::OnDataSent, this, std::placeholders::_1, std::placeholders::_2));

0 commit comments

Comments
 (0)