Description
Describe the bug
Unable to upload S3 object using S3crt API Aws::S3Crt::S3CrtClient::PutObject with custom metadata when the upload size is greater than the configured part size in Aws::S3Crt::ClientConfiguration::partSize
return the error message PutObject error:Unable to parse ExceptionName: InvalidArgument Message: Metadata cannot be specified in this context.
Expected Behavior
The correct behaviour, it should put s3 object with metadata.
Similar like when data size < part size (Used Default 5MB = 5 * 1024 * 1024) see below
- File to be upload of size 4.8 MB
[root@434f62a64297 aws-sdk-cpp]# ls -lh chunk0.ttl
-rw-r--r-- 1 root root 4.8M Apr 8 06:03 chunk0.ttl
- Object uploaded successfully
[root@434f62a64297 aws-sdk-cpp]# ./aws_s3_crt_putobject_w_metadata chunk0.ttl
Putting object s3://my-bucket/test/my_object
Object s3://my-bucket/test/my_object uploaded
Current Behavior
The current observation is below
- File to be upload of size 15MB
[root@434f62a64297 aws-sdk-cpp]# ls -lh chunk1.ttl
-rw-r--r-- 1 root root 15M Apr 8 06:00 chunk1.ttl
- Received below exception
[root@434f62a64297 aws-sdk-cpp]# ./aws_s3_crt_putobject_w_metadata chunk1.ttl
Putting object s3://my-bucket/test/my_object
PutObject error:Unable to parse ExceptionName: InvalidArgument Message: Metadata cannot be specified in this context.
Reproduction Steps
It is easily reproducible using below code snippet
#include <iostream>
#include <string>
#include <fstream>
#include <aws/core/Aws.h>
#include <aws/core/utils/memory/stl/AWSStringStream.h>
#include <aws/core/utils/logging/CRTLogSystem.h>
#include <aws/s3-crt/S3CrtClient.h>
#include <aws/s3-crt/model/PutObjectRequest.h>
static const char ALLOCATION_TAG[] = "s3-crt-putobject-metadata";
int main(int argc, char* argv[])
{
if(argc == 2)
{
Aws::SDKOptions options;
options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Trace;
options.loggingOptions.crt_logger_create_fn = []() {
return Aws::MakeShared<Aws::Utils::Logging::DefaultCRTLogSystem>(
ALLOCATION_TAG, Aws::Utils::Logging::LogLevel::Trace);
};
Aws::InitAPI(options);
{
const double throughput_target_gbps = 5;
Aws::S3Crt::ClientConfiguration config;
config.region = Aws::Region::US_EAST_1;
config.throughputTargetGbps = 5;
config.partSize = 5 * 1024 * 1024;
Aws::S3Crt::S3CrtClient s3CrtClient(config);
Aws::String bucket = "my-bucket";
Aws::String objectKey = "test/my_object";
std::cout << "Putting object s3://" << bucket << "/" << objectKey << std::endl;
Aws::S3Crt::Model::PutObjectRequest request;
request.SetBucket(bucket);
request.SetKey(objectKey);
request.AddMetadata("key1", "value1");
const char* filename = argv[1];
std::shared_ptr<Aws::IOStream> stream = Aws::MakeShared<Aws::FStream>(ALLOCATION_TAG,
filename, std::ios_base::in | std::ios_base::binary);
if (stream->good())
{
request.SetBody(stream);
Aws::S3Crt::Model::PutObjectOutcome outcome = s3CrtClient.PutObject(request);
if(outcome.IsSuccess())
std::cout << "Object s3://" << bucket << "/" << objectKey << " uploaded" << std::endl;
else
std::cerr << "PutObject error:" << outcome.GetError().GetMessage() << std::endl;
}
else
std::cerr << "failed to open file: " << filename << std::endl;
}
Aws::ShutdownAPI(options);
}
else
std::cout << argv[0] << " <file to be upload>"<< std::endl;
return 0;
}
build command
g++ -o aws_s3_crt_putobject_w_metadata aws_s3_crt_putobject_w_metadata.cpp -I<path>/include/ -L/<path>/lib64/ -laws-cpp-sdk-s3-crt -laws-cpp-sdk-core -lssl -lcrypto -lpthread
Please note that replace include and library path in above command
Possible Solution
No response
Additional Information/Context
None
AWS CPP SDK version used
AWS SDK for C++ 1.9.230
Compiler and Version used
g++ (GCC) 11.2.0
Operating System and version
Linux 434f62a64297 5.13.0-39-generic #44~20.04.1-Ubuntu SMP Thu Mar 24 16:43:35 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux