Skip to content

Commit c072458

Browse files
elekbharatviswa504
elek
authored and
bharatviswa504
committed
HDDS-1213. Support plain text S3 MPU initialization request.
1 parent 341c076 commit c072458

File tree

4 files changed

+139
-74
lines changed

4 files changed

+139
-74
lines changed

‎hadoop-ozone/dist/src/main/smoketest/commonlib.robot‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,10 @@ Execute and checkrc
2828
${rc} ${output} = Run And Return Rc And Output ${command}
2929
Log ${output}
3030
Should Be Equal As Integers ${rc} ${expected_error_code}
31-
[return] ${output}
31+
[return] ${output}
32+
33+
Compare files
34+
[arguments] ${file1} ${file2}
35+
${checksumbefore} = Execute md5sum ${file1} | awk '{print $1}'
36+
${checksumafter} = Execute md5sum ${file2} | awk '{print $1}'
37+
Should Be Equal ${checksumbefore} ${checksumafter}

‎hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot‎

Lines changed: 65 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,9 @@ Resource commonawslib.robot
2222
Test Setup Setup s3 tests
2323

2424
*** Keywords ***
25-
Create Random file for mac
26-
Execute dd if=/dev/urandom of=/tmp/part1 bs=1m count=5
27-
28-
Create Random file for linux
29-
Execute dd if=/dev/urandom of=/tmp/part1 bs=1M count=5
25+
Create Random file
26+
[arguments] ${size_in_megabytes}
27+
Execute dd if=/dev/urandom of=/tmp/part1 bs=1048576 count=${size_in_megabytes}
3028

3129

3230
*** Variables ***
@@ -54,16 +52,13 @@ Test Multipart Upload
5452
# upload we get error entity too small. So, considering further complete
5553
# multipart upload, uploading each part as 5MB file, exception is for last part
5654

57-
${system} = Evaluate platform.system() platform
58-
Run Keyword if '${system}' == 'Darwin' Create Random file for mac
59-
Run Keyword if '${system}' == 'Linux' Create Random file for linux
60-
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
61-
Should contain ${result} ETag
55+
Run Keyword Create Random file 5
56+
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
57+
Should contain ${result} ETag
6258
# override part
63-
Run Keyword if '${system}' == 'Darwin' Create Random file for mac
64-
Run Keyword if '${system}' == 'Linux' Create Random file for linux
65-
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
66-
Should contain ${result} ETag
59+
Run Keyword Create Random file 5
60+
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
61+
Should contain ${result} ETag
6762

6863

6964
Test Multipart Upload Complete
@@ -74,17 +69,15 @@ Test Multipart Upload Complete
7469
Should contain ${result} UploadId
7570

7671
#upload parts
77-
${system} = Evaluate platform.system() platform
78-
Run Keyword if '${system}' == 'Darwin' Create Random file for mac
79-
Run Keyword if '${system}' == 'Linux' Create Random file for linux
80-
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
81-
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
82-
Should contain ${result} ETag
72+
Run Keyword Create Random file 5
73+
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
74+
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
75+
Should contain ${result} ETag
8376

8477
Execute echo "Part2" > /tmp/part2
85-
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
86-
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
87-
Should contain ${result} ETag
78+
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
79+
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
80+
Should contain ${result} ETag
8881

8982
#complete multipart upload
9083
${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey1 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
@@ -94,11 +87,8 @@ Test Multipart Upload Complete
9487

9588
#read file and check the key
9689
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key multipartKey1 /tmp/multipartKey1.result
97-
Execute cat /tmp/part1 /tmp/part2 >> /tmp/multipartkey1
98-
${checksumbefore} = Execute md5sum /tmp/multipartkey1 | awk '{print $1}'
99-
${checksumafter} = Execute md5sum /tmp/multipartKey1.result | awk '{print $1}'
100-
Should Be Equal ${checksumbefore} ${checksumafter}
101-
90+
Execute cat /tmp/part1 /tmp/part2 >> /tmp/multipartKey1
91+
Compare files /tmp/multipartKey1 /tmp/multipartKey1.result
10292

10393
Test Multipart Upload Complete Entity too small
10494
${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key multipartKey2
@@ -109,14 +99,14 @@ Test Multipart Upload Complete Entity too small
10999

110100
#upload parts
111101
Execute echo "Part1" > /tmp/part1
112-
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
113-
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
114-
Should contain ${result} ETag
102+
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
103+
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
104+
Should contain ${result} ETag
115105

116106
Execute echo "Part2" > /tmp/part2
117-
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
118-
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
119-
Should contain ${result} ETag
107+
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
108+
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
109+
Should contain ${result} ETag
120110

121111
#complete multipart upload
122112
${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey2 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' 255
@@ -132,14 +122,14 @@ Test Multipart Upload Complete Invalid part
132122

133123
#upload parts
134124
Execute echo "Part1" > /tmp/part1
135-
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
136-
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
137-
Should contain ${result} ETag
125+
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
126+
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
127+
Should contain ${result} ETag
138128

139129
Execute echo "Part2" > /tmp/part2
140-
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
141-
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
142-
Should contain ${result} ETag
130+
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
131+
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
132+
Should contain ${result} ETag
143133

144134
#complete multipart upload
145135
${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255
@@ -158,9 +148,9 @@ Test abort Multipart upload with invalid uploadId
158148
${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id "random" 255
159149

160150
Upload part with Incorrect uploadID
161-
Execute echo "Multipart upload" > /tmp/testfile
162-
${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255
163-
Should contain ${result} NoSuchUpload
151+
Execute echo "Multipart upload" > /tmp/testfile
152+
${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255
153+
Should contain ${result} NoSuchUpload
164154

165155
Test list parts
166156
#initiate multipart upload
@@ -171,37 +161,42 @@ Test list parts
171161
Should contain ${result} UploadId
172162

173163
#upload parts
174-
${system} = Evaluate platform.system() platform
175-
Run Keyword if '${system}' == 'Darwin' Create Random file for mac
176-
Run Keyword if '${system}' == 'Linux' Create Random file for linux
177-
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
178-
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
179-
Should contain ${result} ETag
180-
181-
Execute echo "Part2" > /tmp/part2
182-
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
183-
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
184-
Should contain ${result} ETag
164+
Run Keyword Create Random file 5
165+
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
166+
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
167+
Should contain ${result} ETag
168+
169+
Execute echo "Part2" > /tmp/part2
170+
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
171+
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
172+
Should contain ${result} ETag
185173

186174
#list parts
187-
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID}
188-
${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
189-
${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[1].ETag' 0
190-
Should Be equal ${part1} ${eTag1}
191-
Should contain ${part2} ${eTag2}
192-
Should contain ${result} STANDARD
175+
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID}
176+
${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
177+
${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[1].ETag' 0
178+
Should Be equal ${part1} ${eTag1}
179+
Should contain ${part2} ${eTag2}
180+
Should contain ${result} STANDARD
193181

194182
#list parts with max-items and next token
195-
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1
196-
${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
197-
${token} = Execute and checkrc echo '${result}' | jq -r '.NextToken' 0
198-
Should Be equal ${part1} ${eTag1}
199-
Should contain ${result} STANDARD
183+
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1
184+
${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
185+
${token} = Execute and checkrc echo '${result}' | jq -r '.NextToken' 0
186+
Should Be equal ${part1} ${eTag1}
187+
Should contain ${result} STANDARD
200188

201-
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token ${token}
202-
${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
203-
Should Be equal ${part2} ${eTag2}
204-
Should contain ${result} STANDARD
189+
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token ${token}
190+
${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
191+
Should Be equal ${part2} ${eTag2}
192+
Should contain ${result} STANDARD
205193

206194
#finally abort it
207-
${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} 0
195+
${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} 0
196+
197+
Test Multipart Upload with the simplified aws s3 cp API
198+
Create Random file 22
199+
Execute AWSS3Cli cp /tmp/part1 s3://${BUCKET}/mpyawscli
200+
Execute AWSS3Cli cp s3://${BUCKET}/mpyawscli /tmp/part1.result
201+
Execute AWSS3Cli rm s3://${BUCKET}/mpyawscli
202+
Compare files /tmp/part1 /tmp/part1.result

‎hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,7 @@ Put object to s3
4242
#This test depends on the previous test case. Can't be executes alone
4343
Get object from s3
4444
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 /tmp/testfile.result
45-
${checksumbefore} = Execute md5sum /tmp/testfile | awk '{print $1}'
46-
${checksumafter} = Execute md5sum /tmp/testfile.result | awk '{print $1}'
47-
Should Be Equal ${checksumbefore} ${checksumafter}
45+
Compare files /tmp/testfile /tmp/testfile.result
4846

4947
Get Partial object from s3 with both start and endoffset
5048
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-4 /tmp/testfile1.result

‎hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PlainTextMultipartUploadReader.java‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
* <p>
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
* <p>
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.ozone.s3.endpoint;
19+
20+
import javax.ws.rs.Consumes;
21+
import javax.ws.rs.WebApplicationException;
22+
import javax.ws.rs.core.MediaType;
23+
import javax.ws.rs.core.MultivaluedMap;
24+
import javax.ws.rs.ext.MessageBodyReader;
25+
import javax.ws.rs.ext.Provider;
26+
import java.io.IOException;
27+
import java.io.InputStream;
28+
import java.lang.annotation.Annotation;
29+
import java.lang.reflect.Type;
30+
31+
/**
32+
* Body reader to accept plain text MPU.
33+
* <p>
34+
* Aws s3 api sends a multipartupload request with the content type
35+
* 'text/plain' in case of using 'aws s3 cp' (instead of aws s3api).
36+
* <p>
37+
* Our generic ObjectEndpoint.multipartUpload has a
38+
* CompleteMultipartUploadRequest parameter, which is required only for the
39+
* completion request.
40+
* <p>
41+
* But JaxRS tries to parse it from the body for the requests and in case of
42+
* text/plain requests this parsing is failed. This simple BodyReader enables
43+
* to parse an empty text/plain message and return with an empty completion
44+
* request.
45+
*/
46+
@Provider
47+
@Consumes("text/plain")
48+
public class PlainTextMultipartUploadReader
49+
implements MessageBodyReader<CompleteMultipartUploadRequest> {
50+
51+
@Override
52+
public boolean isReadable(Class<?> type, Type genericType,
53+
Annotation[] annotations, MediaType mediaType) {
54+
return type.equals(CompleteMultipartUploadRequest.class)
55+
&& mediaType.equals(MediaType.TEXT_PLAIN_TYPE);
56+
}
57+
58+
@Override
59+
public CompleteMultipartUploadRequest readFrom(
60+
Class<CompleteMultipartUploadRequest> type, Type genericType,
61+
Annotation[] annotations, MediaType mediaType,
62+
MultivaluedMap<String, String> httpHeaders, InputStream entityStream)
63+
throws IOException, WebApplicationException {
64+
return new CompleteMultipartUploadRequest();
65+
}
66+
}

0 commit comments

Comments
 (0)