Skip to content

Commit 519f68c

Browse files
pjfanningpjfanning
authored
HADOOP-18890. Remove use of okhttp in runtime code (#6057)
 Contributed by PJ Fanning
1 parent 7540fb1 commit 519f68c

File tree

11 files changed

+117
-178
lines changed

11 files changed

+117
-178
lines changed

‎LICENSE-binary‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -243,8 +243,6 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
243243
com.google.j2objc:j2objc-annotations:1.3
244244
com.microsoft.azure:azure-storage:7.0.1
245245
com.nimbusds:nimbus-jose-jwt:9.8.1
246-
com.squareup.okhttp3:okhttp:4.10.0
247-
com.squareup.okio:okio:3.4.0
248246
com.yammer.metrics:metrics-core:2.2.0
249247
com.zaxxer:HikariCP-java7:2.4.12
250248
commons-beanutils:commons-beanutils:1.9.4
@@ -361,8 +359,6 @@ org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.51.v20230217
361359
org.apache.zookeeper:zookeeper:3.6.3
362360
org.ehcache:ehcache:3.3.1
363361
org.ini4j:ini4j:0.5.4
364-
org.jetbrains.kotlin:kotlin-stdlib:1.4.10
365-
org.jetbrains.kotlin:kotlin-stdlib-common:1.4.10
366362
org.lz4:lz4-java:1.7.1
367363
org.objenesis:objenesis:2.6
368364
org.xerial.snappy:snappy-java:1.1.10.1

‎NOTICE-binary‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -334,19 +334,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
334334
See the License for the specific language governing permissions and
335335
limitations under the License.
336336

337-
-----------------------------------------------------------------------
338-
339-
This product contains a modified portion of 'OkHttp', an open source
340-
HTTP & SPDY client for Android and Java applications, which can be obtained
341-
at:
342-
343-
* LICENSE:
344-
* okhttp/third_party/okhttp/LICENSE (Apache License 2.0)
345-
* HOMEPAGE:
346-
* https://github.com/square/okhttp
347-
* LOCATION_IN_GRPC:
348-
* okhttp/third_party/okhttp
349-
350337
This product contains a modified portion of 'Netty', an open source
351338
networking library, which can be obtained at:
352339

‎hadoop-client-modules/hadoop-client/pom.xml‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -114,18 +114,6 @@
114114
<groupId>org.eclipse.jetty</groupId>
115115
<artifactId>jetty-server</artifactId>
116116
</exclusion>
117-
<exclusion>
118-
<groupId>org.jetbrains.kotlin</groupId>
119-
<artifactId>kotlin-stdlib</artifactId>
120-
</exclusion>
121-
<exclusion>
122-
<groupId>org.jetbrains.kotlin</groupId>
123-
<artifactId>kotlin-stdlib-common</artifactId>
124-
</exclusion>
125-
<exclusion>
126-
<groupId>com.squareup.okhttp3</groupId>
127-
<artifactId>okhttp</artifactId>
128-
</exclusion>
129117
<exclusion>
130118
<groupId>com.sun.jersey</groupId>
131119
<artifactId>jersey-core</artifactId>

‎hadoop-common-project/hadoop-common/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -374,8 +374,8 @@
374374
<scope>test</scope>
375375
</dependency>
376376
<dependency>
377-
<groupId>com.squareup.okio</groupId>
378-
<artifactId>okio-jvm</artifactId>
377+
<groupId>org.jetbrains.kotlin</groupId>
378+
<artifactId>kotlin-stdlib-jdk8</artifactId>
379379
<scope>test</scope>
380380
</dependency>
381381
<dependency>

‎hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -93,17 +93,4 @@
9393
<Bug pattern="EI_EXPOSE_REP" />
9494
</Match>
9595

96-
<!--okhttp classes from Kotlin are not analysed for NP check. -->
97-
<Match>
98-
<Class name="org.apache.hadoop.hdfs.web.oauth2.ConfRefreshTokenBasedAccessTokenProvider" />
99-
<Method name="refresh" />
100-
<Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
101-
</Match>
102-
103-
<Match>
104-
<Class name="org.apache.hadoop.hdfs.web.oauth2.CredentialBasedAccessTokenProvider" />
105-
<Method name="refresh" />
106-
<Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
107-
</Match>
108-
10996
</FindBugsFilter>

‎hadoop-hdfs-project/hadoop-hdfs-client/pom.xml‎

Lines changed: 8 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -34,28 +34,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
3434
</properties>
3535

3636
<dependencies>
37-
<dependency>
38-
<groupId>com.squareup.okhttp3</groupId>
39-
<artifactId>okhttp</artifactId>
40-
<exclusions>
41-
<exclusion>
42-
<groupId>com.squareup.okio</groupId>
43-
<artifactId>okio-jvm</artifactId>
44-
</exclusion>
45-
</exclusions>
46-
</dependency>
47-
<dependency>
48-
<groupId>com.squareup.okio</groupId>
49-
<artifactId>okio-jvm</artifactId>
50-
</dependency>
51-
<dependency>
52-
<groupId>org.jetbrains.kotlin</groupId>
53-
<artifactId>kotlin-stdlib</artifactId>
54-
</dependency>
55-
<dependency>
56-
<groupId>org.jetbrains.kotlin</groupId>
57-
<artifactId>kotlin-stdlib-common</artifactId>
58-
</dependency>
5937
<dependency>
6038
<groupId>org.apache.hadoop</groupId>
6139
<artifactId>hadoop-common</artifactId>
@@ -71,6 +49,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
7149
</exclusion>
7250
</exclusions>
7351
</dependency>
52+
<dependency>
53+
<groupId>org.apache.httpcomponents</groupId>
54+
<artifactId>httpclient</artifactId>
55+
</dependency>
56+
<dependency>
57+
<groupId>org.apache.httpcomponents</groupId>
58+
<artifactId>httpcore</artifactId>
59+
</dependency>
7460
<dependency>
7561
<groupId>junit</groupId>
7662
<artifactId>junit</artifactId>

‎hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java‎

Lines changed: 43 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,28 @@
1919
package org.apache.hadoop.hdfs.web.oauth2;
2020

2121
import java.io.IOException;
22+
import java.nio.charset.StandardCharsets;
23+
import java.util.ArrayList;
24+
import java.util.List;
2225
import java.util.Map;
23-
import java.util.concurrent.TimeUnit;
24-
25-
import okhttp3.OkHttpClient;
26-
import okhttp3.Request;
27-
import okhttp3.RequestBody;
28-
import okhttp3.Response;
2926

3027
import org.apache.hadoop.classification.InterfaceAudience;
3128
import org.apache.hadoop.classification.InterfaceStability;
3229
import org.apache.hadoop.conf.Configuration;
3330
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
3431
import org.apache.hadoop.util.JsonSerialization;
3532
import org.apache.hadoop.util.Timer;
33+
import org.apache.http.HttpHeaders;
3634
import org.apache.http.HttpStatus;
35+
import org.apache.http.NameValuePair;
36+
import org.apache.http.client.config.RequestConfig;
37+
import org.apache.http.client.entity.UrlEncodedFormEntity;
38+
import org.apache.http.client.methods.CloseableHttpResponse;
39+
import org.apache.http.client.methods.HttpPost;
40+
import org.apache.http.impl.client.CloseableHttpClient;
41+
import org.apache.http.impl.client.HttpClientBuilder;
42+
import org.apache.http.message.BasicNameValuePair;
43+
import org.apache.http.util.EntityUtils;
3744

3845
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
3946
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
@@ -103,34 +110,37 @@ public synchronized String getAccessToken() throws IOException {
103110
}
104111

105112
void refresh() throws IOException {
106-
OkHttpClient client =
107-
new OkHttpClient.Builder().connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT,
108-
TimeUnit.MILLISECONDS)
109-
.readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
110-
.build();
111-
112-
String bodyString =
113-
Utils.postBody(GRANT_TYPE, REFRESH_TOKEN, REFRESH_TOKEN, refreshToken, CLIENT_ID, clientId);
114-
115-
RequestBody body = RequestBody.create(bodyString, URLENCODED);
116-
117-
Request request = new Request.Builder().url(refreshURL).post(body).build();
118-
try (Response response = client.newCall(request).execute()) {
119-
if (!response.isSuccessful()) {
120-
throw new IOException("Unexpected code " + response);
121-
}
122-
if (response.code() != HttpStatus.SC_OK) {
123-
throw new IllegalArgumentException(
124-
"Received invalid http response: " + response.code() + ", text = "
125-
+ response.toString());
113+
final List<NameValuePair> pairs = new ArrayList<>();
114+
pairs.add(new BasicNameValuePair(GRANT_TYPE, REFRESH_TOKEN));
115+
pairs.add(new BasicNameValuePair(REFRESH_TOKEN, refreshToken));
116+
pairs.add(new BasicNameValuePair(CLIENT_ID, clientId));
117+
final RequestConfig config = RequestConfig.custom()
118+
.setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
119+
.setConnectionRequestTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
120+
.setSocketTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
121+
.build();
122+
try (CloseableHttpClient client =
123+
HttpClientBuilder.create().setDefaultRequestConfig(config).build()) {
124+
final HttpPost httpPost = new HttpPost(refreshURL);
125+
httpPost.setEntity(new UrlEncodedFormEntity(pairs, StandardCharsets.UTF_8));
126+
httpPost.setHeader(HttpHeaders.CONTENT_TYPE, URLENCODED);
127+
try (CloseableHttpResponse response = client.execute(httpPost)) {
128+
final int statusCode = response.getStatusLine().getStatusCode();
129+
if (statusCode != HttpStatus.SC_OK) {
130+
throw new IllegalArgumentException(
131+
"Received invalid http response: " + statusCode + ", text = " +
132+
EntityUtils.toString(response.getEntity()));
133+
}
134+
Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(
135+
EntityUtils.toString(response.getEntity()));
136+
137+
String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
138+
accessTokenTimer.setExpiresIn(newExpiresIn);
139+
140+
accessToken = responseBody.get(ACCESS_TOKEN).toString();
126141
}
127-
128-
Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(response.body().string());
129-
130-
String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
131-
accessTokenTimer.setExpiresIn(newExpiresIn);
132-
133-
accessToken = responseBody.get(ACCESS_TOKEN).toString();
142+
} catch (RuntimeException e) {
143+
throw new IOException("Exception while refreshing access token", e);
134144
} catch (Exception e) {
135145
throw new IOException("Exception while refreshing access token", e);
136146
}

‎hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java‎

Lines changed: 42 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,28 @@
1919
package org.apache.hadoop.hdfs.web.oauth2;
2020

2121
import java.io.IOException;
22+
import java.nio.charset.StandardCharsets;
23+
import java.util.ArrayList;
24+
import java.util.List;
2225
import java.util.Map;
23-
import java.util.concurrent.TimeUnit;
24-
25-
import okhttp3.OkHttpClient;
26-
import okhttp3.Request;
27-
import okhttp3.RequestBody;
28-
import okhttp3.Response;
2926

3027
import org.apache.hadoop.classification.InterfaceAudience;
3128
import org.apache.hadoop.classification.InterfaceStability;
3229
import org.apache.hadoop.conf.Configuration;
3330
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
3431
import org.apache.hadoop.util.JsonSerialization;
3532
import org.apache.hadoop.util.Timer;
33+
import org.apache.http.HttpHeaders;
3634
import org.apache.http.HttpStatus;
35+
import org.apache.http.NameValuePair;
36+
import org.apache.http.client.config.RequestConfig;
37+
import org.apache.http.client.entity.UrlEncodedFormEntity;
38+
import org.apache.http.client.methods.CloseableHttpResponse;
39+
import org.apache.http.client.methods.HttpPost;
40+
import org.apache.http.impl.client.CloseableHttpClient;
41+
import org.apache.http.impl.client.HttpClientBuilder;
42+
import org.apache.http.message.BasicNameValuePair;
43+
import org.apache.http.util.EntityUtils;
3744

3845
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_CLIENT_ID_KEY;
3946
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.OAUTH_REFRESH_URL_KEY;
@@ -97,38 +104,37 @@ public synchronized String getAccessToken() throws IOException {
97104
}
98105

99106
void refresh() throws IOException {
100-
OkHttpClient client = new OkHttpClient.Builder()
101-
.connectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
102-
.readTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS)
103-
.build();
104-
105-
String bodyString = Utils.postBody(CLIENT_SECRET, getCredential(),
106-
GRANT_TYPE, CLIENT_CREDENTIALS,
107-
CLIENT_ID, clientId);
108-
109-
RequestBody body = RequestBody.create(bodyString, URLENCODED);
110-
111-
Request request = new Request.Builder()
112-
.url(refreshURL)
113-
.post(body)
107+
final List<NameValuePair> pairs = new ArrayList<>();
108+
pairs.add(new BasicNameValuePair(CLIENT_SECRET, getCredential()));
109+
pairs.add(new BasicNameValuePair(GRANT_TYPE, CLIENT_CREDENTIALS));
110+
pairs.add(new BasicNameValuePair(CLIENT_ID, clientId));
111+
final RequestConfig config = RequestConfig.custom()
112+
.setConnectTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
113+
.setConnectionRequestTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
114+
.setSocketTimeout(URLConnectionFactory.DEFAULT_SOCKET_TIMEOUT)
114115
.build();
115-
try (Response response = client.newCall(request).execute()) {
116-
if (!response.isSuccessful()) {
117-
throw new IOException("Unexpected code " + response);
118-
}
119-
120-
if (response.code() != HttpStatus.SC_OK) {
121-
throw new IllegalArgumentException("Received invalid http response: "
122-
+ response.code() + ", text = " + response.toString());
116+
try (CloseableHttpClient client =
117+
HttpClientBuilder.create().setDefaultRequestConfig(config).build()) {
118+
final HttpPost httpPost = new HttpPost(refreshURL);
119+
httpPost.setEntity(new UrlEncodedFormEntity(pairs, StandardCharsets.UTF_8));
120+
httpPost.setHeader(HttpHeaders.CONTENT_TYPE, URLENCODED);
121+
try (CloseableHttpResponse response = client.execute(httpPost)) {
122+
final int statusCode = response.getStatusLine().getStatusCode();
123+
if (statusCode != HttpStatus.SC_OK) {
124+
throw new IllegalArgumentException(
125+
"Received invalid http response: " + statusCode + ", text = " +
126+
EntityUtils.toString(response.getEntity()));
127+
}
128+
Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(
129+
EntityUtils.toString(response.getEntity()));
130+
131+
String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
132+
timer.setExpiresIn(newExpiresIn);
133+
134+
accessToken = responseBody.get(ACCESS_TOKEN).toString();
123135
}
124-
125-
Map<?, ?> responseBody = JsonSerialization.mapReader().readValue(
126-
response.body().string());
127-
128-
String newExpiresIn = responseBody.get(EXPIRES_IN).toString();
129-
timer.setExpiresIn(newExpiresIn);
130-
131-
accessToken = responseBody.get(ACCESS_TOKEN).toString();
136+
} catch (RuntimeException e) {
137+
throw new IOException("Unable to obtain access token from credential", e);
132138
} catch (Exception e) {
133139
throw new IOException("Unable to obtain access token from credential", e);
134140
}

‎hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/OAuth2Constants.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
*/
1919
package org.apache.hadoop.hdfs.web.oauth2;
2020

21-
import okhttp3.MediaType;
2221
import org.apache.hadoop.classification.InterfaceAudience;
2322
import org.apache.hadoop.classification.InterfaceStability;
2423

@@ -30,8 +29,8 @@
3029
public final class OAuth2Constants {
3130
private OAuth2Constants() { /** Private constructor. **/ }
3231

33-
public static final MediaType URLENCODED
34-
= MediaType.parse("application/x-www-form-urlencoded; charset=utf-8");
32+
public static final String URLENCODED
33+
= "application/x-www-form-urlencoded; charset=utf-8";
3534

3635
/* Constants for OAuth protocol */
3736
public static final String ACCESS_TOKEN = "access_token";

‎hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,16 @@
199199
<artifactId>bcprov-jdk15on</artifactId>
200200
<scope>test</scope>
201201
</dependency>
202+
<dependency>
203+
<groupId>com.squareup.okhttp3</groupId>
204+
<artifactId>mockwebserver</artifactId>
205+
<scope>test</scope>
206+
</dependency>
207+
<dependency>
208+
<groupId>org.jetbrains.kotlin</groupId>
209+
<artifactId>kotlin-stdlib-jdk8</artifactId>
210+
<scope>test</scope>
211+
</dependency>
202212
</dependencies>
203213

204214
<build>

0 commit comments

Comments
 (0)