Skip to content

Commit 2b63512

Browse files
author
Sammi Chen
committed
HADOOP-15671. AliyunOSS: Support Assume Roles in AliyunOSS. Contributed by Jinhu Wu.
1 parent 93b0f54 commit 2b63512

File tree

7 files changed

+248
-12
lines changed

7 files changed

+248
-12
lines changed

‎hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,8 @@ public synchronized void close() throws IOException {
120120
if (null == partETags) {
121121
throw new IOException("Failed to multipart upload to oss, abort it.");
122122
}
123-
store.completeMultipartUpload(key, uploadId, partETags);
123+
store.completeMultipartUpload(key, uploadId,
124+
new ArrayList<>(partETags));
124125
}
125126
} finally {
126127
removePartFiles();
@@ -129,7 +130,7 @@ public synchronized void close() throws IOException {
129130
}
130131

131132
@Override
132-
public void write(int b) throws IOException {
133+
public synchronized void write(int b) throws IOException {
133134
singleByte[0] = (byte)b;
134135
write(singleByte, 0, 1);
135136
}

‎hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ public void initialize(URI uri, Configuration conf,
149149
"null or empty. Please set proper endpoint with 'fs.oss.endpoint'.");
150150
}
151151
CredentialsProvider provider =
152-
AliyunOSSUtils.getCredentialsProvider(conf);
152+
AliyunOSSUtils.getCredentialsProvider(uri, conf);
153153
ossClient = new OSSClient(endPoint, provider, clientConf);
154154
uploadPartSize = AliyunOSSUtils.getMultipartSizeProperty(conf,
155155
MULTIPART_UPLOAD_PART_SIZE_KEY, MULTIPART_UPLOAD_PART_SIZE_DEFAULT);
@@ -168,6 +168,8 @@ public void initialize(URI uri, Configuration conf,
168168
multipartThreshold = 1024 * 1024 * 1024;
169169
}
170170

171+
bucketName = uri.getHost();
172+
171173
String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT);
172174
if (StringUtils.isNotEmpty(cannedACLName)) {
173175
CannedAccessControlList cannedACL =
@@ -176,7 +178,6 @@ public void initialize(URI uri, Configuration conf,
176178
}
177179

178180
maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
179-
bucketName = uri.getHost();
180181
}
181182

182183
/**

‎hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import java.io.File;
2222
import java.io.IOException;
23+
import java.net.URI;
2324

2425
import com.aliyun.oss.common.auth.CredentialsProvider;
2526
import com.google.common.base.Preconditions;
@@ -95,13 +96,14 @@ public static long calculatePartSize(long contentLength, long minPartSize) {
9596
* Create credential provider specified by configuration, or create default
9697
* credential provider if not specified.
9798
*
99+
* @param uri uri passed by caller
98100
* @param conf configuration
99101
* @return a credential provider
100102
* @throws IOException on any problem. Class construction issues may be
101103
* nested inside the IOE.
102104
*/
103-
public static CredentialsProvider getCredentialsProvider(Configuration conf)
104-
throws IOException {
105+
public static CredentialsProvider getCredentialsProvider(
106+
URI uri, Configuration conf) throws IOException {
105107
CredentialsProvider credentials;
106108

107109
String className = conf.getTrimmed(CREDENTIALS_PROVIDER_KEY);
@@ -117,7 +119,7 @@ public static CredentialsProvider getCredentialsProvider(Configuration conf)
117119
try {
118120
credentials =
119121
(CredentialsProvider)credClass.getDeclaredConstructor(
120-
Configuration.class).newInstance(conf);
122+
URI.class, Configuration.class).newInstance(uri, conf);
121123
} catch (NoSuchMethodException | SecurityException e) {
122124
credentials =
123125
(CredentialsProvider)credClass.getDeclaredConstructor()

‎hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AssumedRoleCredentialProvider.java‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.aliyun.oss;
20+
21+
import com.aliyun.oss.common.auth.Credentials;
22+
import com.aliyun.oss.common.auth.CredentialsProvider;
23+
import com.aliyun.oss.common.auth.InvalidCredentialsException;
24+
import com.aliyun.oss.common.auth.STSAssumeRoleSessionCredentialsProvider;
25+
import com.aliyuncs.exceptions.ClientException;
26+
import com.aliyuncs.profile.DefaultProfile;
27+
import org.apache.commons.lang3.StringUtils;
28+
import org.apache.hadoop.conf.Configuration;
29+
import org.slf4j.Logger;
30+
import org.slf4j.LoggerFactory;
31+
32+
import java.io.IOException;
33+
import java.net.URI;
34+
35+
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID;
36+
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET;
37+
38+
/**
39+
* Support assumed role credentials for authenticating with Aliyun.
40+
* roleArn is configured in core-site.xml
41+
*/
42+
public class AssumedRoleCredentialProvider implements CredentialsProvider {
43+
private static final Logger LOG =
44+
LoggerFactory.getLogger(AssumedRoleCredentialProvider.class);
45+
public static final String NAME
46+
= "org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider";
47+
private Credentials credentials;
48+
private String roleArn;
49+
private long duration;
50+
private String stsEndpoint;
51+
private String sessionName;
52+
private double expiredFactor;
53+
private STSAssumeRoleSessionCredentialsProvider stsCredentialsProvider;
54+
55+
public AssumedRoleCredentialProvider(URI uri, Configuration conf) {
56+
roleArn = conf.getTrimmed(Constants.ROLE_ARN, "");
57+
if (StringUtils.isEmpty(roleArn)) {
58+
throw new InvalidCredentialsException(
59+
"fs.oss.assumed.role.arn is empty");
60+
}
61+
62+
duration = conf.getLong(Constants.ASSUMED_ROLE_DURATION,
63+
Constants.ASSUMED_ROLE_DURATION_DEFAULT);
64+
65+
expiredFactor = conf.getDouble(Constants.ASSUMED_ROLE_STS_EXPIRED_FACTOR,
66+
Constants.ASSUMED_ROLE_STS_EXPIRED_FACTOR_DEFAULT);
67+
68+
stsEndpoint = conf.getTrimmed(Constants.ASSUMED_ROLE_STS_ENDPOINT, "");
69+
if (StringUtils.isEmpty(stsEndpoint)) {
70+
throw new InvalidCredentialsException(
71+
"fs.oss.assumed.role.sts.endpoint is empty");
72+
}
73+
74+
sessionName = conf.getTrimmed(Constants.ASSUMED_ROLE_SESSION_NAME, "");
75+
76+
String accessKeyId;
77+
String accessKeySecret;
78+
try {
79+
accessKeyId = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_ID);
80+
accessKeySecret = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_SECRET);
81+
} catch (IOException e) {
82+
throw new InvalidCredentialsException(e);
83+
}
84+
85+
try {
86+
DefaultProfile.addEndpoint("", "", "Sts", stsEndpoint);
87+
} catch (ClientException e) {
88+
throw new InvalidCredentialsException(e);
89+
}
90+
91+
stsCredentialsProvider = new STSAssumeRoleSessionCredentialsProvider(
92+
new com.aliyuncs.auth.BasicCredentials(accessKeyId, accessKeySecret),
93+
roleArn, DefaultProfile.getProfile("", accessKeyId, accessKeySecret))
94+
.withExpiredDuration(duration).withExpiredFactor(expiredFactor);
95+
96+
if (!StringUtils.isEmpty(sessionName)) {
97+
stsCredentialsProvider.withRoleSessionName(sessionName);
98+
}
99+
}
100+
101+
@Override
102+
public void setCredentials(Credentials creds) {
103+
throw new InvalidCredentialsException(
104+
"Should not set credentials from external call");
105+
}
106+
107+
@Override
108+
public Credentials getCredentials() {
109+
credentials = stsCredentialsProvider.getCredentials();
110+
if (credentials == null) {
111+
throw new InvalidCredentialsException("Invalid credentials");
112+
}
113+
return credentials;
114+
}
115+
}

‎hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
package org.apache.hadoop.fs.aliyun.oss;
2020

21+
import com.aliyun.oss.common.utils.AuthUtils;
2122
import com.aliyun.oss.common.utils.VersionInfoUtils;
2223

2324
/**
@@ -42,6 +43,27 @@ private Constants() {
4243
public static final String ACCESS_KEY_SECRET = "fs.oss.accessKeySecret";
4344
public static final String SECURITY_TOKEN = "fs.oss.securityToken";
4445

46+
// Assume role configurations
47+
public static final String ROLE_ARN = "fs.oss.assumed.role.arn";
48+
public static final String ASSUMED_ROLE_DURATION =
49+
"fs.oss.assumed.role.session.duration";
50+
// Default session duration(in seconds)
51+
public static final long ASSUMED_ROLE_DURATION_DEFAULT = 30 * 60;
52+
53+
// Expired factor of sts token
54+
// For example, if session duration is 900s and expiredFactor is 0.8
55+
// sts token will be refreshed after 900 * 0.8s
56+
public static final String ASSUMED_ROLE_STS_EXPIRED_FACTOR =
57+
"fs.oss.assumed.role.sts.expiredFactor";
58+
59+
public static final double ASSUMED_ROLE_STS_EXPIRED_FACTOR_DEFAULT =
60+
AuthUtils.DEFAULT_EXPIRED_FACTOR;
61+
62+
public static final String ASSUMED_ROLE_STS_ENDPOINT =
63+
"fs.oss.assumed.role.sts.endpoint";
64+
public static final String ASSUMED_ROLE_SESSION_NAME =
65+
"fs.oss.assumed.role.session.name";
66+
4567
// Number of simultaneous connections to oss
4668
public static final String MAXIMUM_CONNECTIONS_KEY =
4769
"fs.oss.connection.maximum";

‎hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,56 @@ please raise your issues with them.
117117
</description>
118118
</property>
119119

120+
<property>
121+
<name>fs.oss.assumed.role.arn</name>
122+
<description>
123+
Role ARN for the role to be assumed.
124+
Required if the fs.oss.credentials.provider is
125+
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
126+
</description>
127+
</property>
128+
129+
<property>
130+
<name>fs.oss.assumed.role.sts.endpoint</name>
131+
<description>
132+
STS Token Service endpoint.
133+
Required if the fs.oss.credentials.provider is
134+
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
135+
</description>
136+
</property>
137+
138+
<property>
139+
<name>fs.oss.assumed.role.session.name</name>
140+
<value />
141+
<description>
142+
Session name for the assumed role, must be valid characters
143+
according to Aliyun API. It is optional, will be generated by
144+
oss java sdk if it is empty.
145+
Only used if the fs.oss.credentials.provider is
146+
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
147+
</description>
148+
</property>
149+
150+
<property>
151+
<name>fs.oss.assumed.role.session.duration</name>
152+
<value />
153+
<description>
154+
Duration of assumed roles before it is expired. Default is 30 minutes.
155+
Only used if the fs.oss.credentials.provider is
156+
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
157+
</description>
158+
</property>
159+
160+
<property>
161+
<name>fs.oss.assumed.role.sts.expiredFactor</name>
162+
<value />
163+
<description>
164+
Sts token will be refreshed after (expiredFactor * duration) seconds.
165+
Only used if the fs.oss.credentials.provider is
166+
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
167+
</description>
168+
</property>
169+
120170
<property>
121171
<name>fs.oss.proxy.host</name>
122172
<description>Hostname of the (optinal) proxy server for Aliyun OSS connection</description>

‎hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java‎

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
package org.apache.hadoop.fs.aliyun.oss;
2020

2121
import com.aliyun.oss.common.auth.Credentials;
22+
import com.aliyun.oss.common.auth.CredentialsProvider;
2223
import com.aliyun.oss.common.auth.InvalidCredentialsException;
2324
import org.apache.hadoop.conf.Configuration;
2425
import org.apache.hadoop.fs.aliyun.oss.contract.AliyunOSSContract;
@@ -27,9 +28,15 @@
2728
import org.junit.Test;
2829

2930
import java.io.IOException;
31+
import java.lang.reflect.InvocationTargetException;
32+
import java.net.URI;
3033

3134
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID;
3235
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET;
36+
import static org.apache.hadoop.fs.aliyun.oss.Constants.ASSUMED_ROLE_SESSION_NAME;
37+
import static org.apache.hadoop.fs.aliyun.oss.Constants.ASSUMED_ROLE_STS_ENDPOINT;
38+
import static org.apache.hadoop.fs.aliyun.oss.Constants.CREDENTIALS_PROVIDER_KEY;
39+
import static org.apache.hadoop.fs.aliyun.oss.Constants.ROLE_ARN;
3340
import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN;
3441

3542
/**
@@ -63,16 +70,54 @@ public void testCredentialMissingAccessKeySecret() throws Throwable {
6370
validateCredential(conf);
6471
}
6572

66-
private void validateCredential(Configuration conf) {
73+
@Test
74+
public void testCredentialMissingRoleArn() throws Throwable {
75+
Configuration conf = new Configuration();
76+
conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME);
77+
conf.set(ROLE_ARN, "");
78+
validateCredential(conf);
79+
}
80+
81+
@Test
82+
public void testCredentialMissingStsEndpoint() throws Throwable {
83+
Configuration conf = new Configuration();
84+
conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME);
85+
conf.set(ASSUMED_ROLE_STS_ENDPOINT, "");
86+
validateCredential(conf);
87+
}
88+
89+
@Test
90+
public void testCredentialInvalidSessionName() throws Throwable {
91+
Configuration conf = new Configuration();
92+
conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME);
93+
conf.set(ASSUMED_ROLE_SESSION_NAME, "hadoop oss");
94+
validateCredential(conf);
95+
}
96+
97+
private void validateCredential(URI uri, Configuration conf) {
6798
try {
68-
AliyunCredentialsProvider provider
69-
= new AliyunCredentialsProvider(conf);
99+
CredentialsProvider provider =
100+
AliyunOSSUtils.getCredentialsProvider(uri, conf);
70101
Credentials credentials = provider.getCredentials();
71102
fail("Expected a CredentialInitializationException, got " + credentials);
72103
} catch (InvalidCredentialsException expected) {
73104
// expected
74105
} catch (IOException e) {
75-
fail("Unexpected exception.");
106+
Throwable cause = e.getCause();
107+
if (cause instanceof InvocationTargetException) {
108+
boolean isInstance =
109+
((InvocationTargetException)cause).getTargetException()
110+
instanceof InvalidCredentialsException;
111+
if (!isInstance) {
112+
fail("Unexpected exception.");
113+
}
114+
} else {
115+
fail("Unexpected exception.");
116+
}
76117
}
77118
}
78-
}
119+
120+
private void validateCredential(Configuration conf) {
121+
validateCredential(null, conf);
122+
}
123+
}

0 commit comments

Comments
 (0)