Skip to content

Commit 2bc7c41

Browse files
jxhan3jxhan3
committed
[S3AG] address comments
1 parent c5c78fc commit 2bc7c41

File tree

5 files changed

+56
-32
lines changed

5 files changed

+56
-32
lines changed

‎hadoop-project/pom.xml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1976,6 +1976,30 @@
19761976
<artifactId>log4j-web</artifactId>
19771977
<version>${log4j2.version}</version>
19781978
</dependency>
1979+
<dependency>
1980+
<groupId>software.amazon.s3.accessgrants</groupId>
1981+
<artifactId>aws-s3-accessgrants-java-plugin</artifactId>
1982+
<version>2.0.0</version>
1983+
<scope>provided</scope>
1984+
<exclusions>
1985+
<exclusion>
1986+
<groupId>software.amazon.awssdk</groupId>
1987+
<artifactId>*</artifactId>
1988+
</exclusion>
1989+
<exclusion>
1990+
<groupId>com.github.ben-manes.caffeine</groupId>
1991+
<artifactId>*</artifactId>
1992+
</exclusion>
1993+
<exclusion>
1994+
<groupId>org.apache.logging.log4j</groupId>
1995+
<artifactId>*</artifactId>
1996+
</exclusion>
1997+
<exclusion>
1998+
<groupId>org.assertj</groupId>
1999+
<artifactId>*</artifactId>
2000+
</exclusion>
2001+
</exclusions>
2002+
</dependency>
19792003
</dependencies>
19802004
</dependencyManagement>
19812005

‎hadoop-tools/hadoop-aws/pom.xml‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -511,26 +511,7 @@
511511
<dependency>
512512
<groupId>software.amazon.s3.accessgrants</groupId>
513513
<artifactId>aws-s3-accessgrants-java-plugin</artifactId>
514-
<version>2.0.0</version>
515514
<scope>provided</scope>
516-
<exclusions>
517-
<exclusion>
518-
<groupId>software.amazon.awssdk</groupId>
519-
<artifactId>*</artifactId>
520-
</exclusion>
521-
<exclusion>
522-
<groupId>com.github.ben-manes.caffeine</groupId>
523-
<artifactId>*</artifactId>
524-
</exclusion>
525-
<exclusion>
526-
<groupId>org.apache.logging.log4j</groupId>
527-
<artifactId>*</artifactId>
528-
</exclusion>
529-
<exclusion>
530-
<groupId>org.assertj</groupId>
531-
<artifactId>*</artifactId>
532-
</exclusion>
533-
</exclusions>
534515
</dependency>
535516
<dependency>
536517
<groupId>org.assertj</groupId>

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ public class DefaultS3ClientFactory extends Configured
9090
protected static final Logger LOG =
9191
LoggerFactory.getLogger(DefaultS3ClientFactory.class);
9292

93-
private static final LogExactlyOnce LOG_EXACTLY_ONCE = new LogExactlyOnce(LOG);
93+
private static final LogExactlyOnce LOG_S3AG_ENABLED = new LogExactlyOnce(LOG);
9494

9595
/**
9696
* A one-off warning of default region chains in use.
@@ -117,7 +117,7 @@ public class DefaultS3ClientFactory extends Configured
117117
"An endpoint cannot set when " + FIPS_ENDPOINT + " is true";
118118

119119
private static final String S3AG_UTIL_CLASSNAME =
120-
"org.apache.hadoop.fs.s3a.tools.S3AccessGrantsUtil";
120+
"org.apache.hadoop.fs.s3a.impl.S3AccessGrantsUtil";
121121
@Override
122122
public S3Client createS3Client(
123123
final URI uri,
@@ -413,24 +413,25 @@ private static Region getS3RegionFromEndpoint(final String endpoint,
413413
applyS3AccessGrantsConfigurations(BuilderT builder, Configuration conf) {
414414
boolean s3agEnabled = conf.getBoolean(AWS_S3_ACCESS_GRANTS_ENABLED, false);
415415
if (!s3agEnabled){
416-
LOG_EXACTLY_ONCE.debug("s3ag plugin is not enabled.");
416+
LOG.debug("S3 Access Grants plugin is not enabled.");
417417
return;
418418
}
419419
try {
420+
LOG_S3AG_ENABLED.info("S3 Access Grants plugin is enabled.");
420421
Class s3agUtil = Class.forName(S3AG_UTIL_CLASSNAME);
421422
Method applyS3agConfig =
422423
s3agUtil.getMethod("applyS3AccessGrantsConfigurations", S3BaseClientBuilder.class, Configuration.class);
423424
applyS3agConfig.invoke(null, builder, conf);
424425
} catch (ClassNotFoundException e) {
425-
LOG_EXACTLY_ONCE.debug(
426+
LOG.debug(
426427
"Class {} is not found exception: {}.",
427428
S3AG_UTIL_CLASSNAME,
428429
e.getStackTrace()
429430
);
430431
} catch (Exception e) {
431-
LOG_EXACTLY_ONCE.debug("{} exception: {})", e.getClass(), e.getStackTrace());
432+
LOG.debug("{} exception: {})", e.getClass(), e.getStackTrace());
432433
} catch (NoClassDefFoundError e) {
433-
LOG_EXACTLY_ONCE.debug(
434+
LOG.debug(
434435
"Class {} is not found error: ",
435436
S3AG_UTIL_CLASSNAME,
436437
e.getStackTrace()

‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/tools/S3AccessGrantsUtil.java‎ renamed to ‎hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AccessGrantsUtil.java‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,22 @@
1-
package org.apache.hadoop.fs.s3a.tools;
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.s3a.impl;
220

321
import org.apache.hadoop.conf.Configuration;
422
import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory;
@@ -15,7 +33,7 @@ public class S3AccessGrantsUtil {
1533
protected static final Logger LOG =
1634
LoggerFactory.getLogger(S3AccessGrantsUtil.class);
1735

18-
private static final LogExactlyOnce LOG_EXACTLY_ONCE = new LogExactlyOnce(LOG);
36+
private static final LogExactlyOnce LOG_S3AG_PLUGIN_INFO = new LogExactlyOnce(LOG);
1937
private static final String S3AG_PLUGIN_CLASSNAME =
2038
"software.amazon.awssdk.s3accessgrants.plugin.S3AccessGrantsPlugin";
2139

@@ -28,10 +46,10 @@ private static boolean checkForS3AGPlugin() {
2846
try {
2947
ClassLoader cl = DefaultS3ClientFactory.class.getClassLoader();
3048
cl.loadClass(S3AG_PLUGIN_CLASSNAME);
31-
LOG.debug("S3AG plugin class {} found", S3AG_PLUGIN_CLASSNAME);
49+
LOG.debug("S3 Access Grants plugin class {} found", S3AG_PLUGIN_CLASSNAME);
3250
return true;
3351
} catch (Exception e) {
34-
LOG.debug("S3AG plugin class {} not found", S3AG_PLUGIN_CLASSNAME, e);
52+
LOG.debug("S3 Access Grants plugin class {} not found", S3AG_PLUGIN_CLASSNAME, e);
3553
return false;
3654
}
3755
}
@@ -52,9 +70,8 @@ private static synchronized boolean isS3AGPluginAvailable() {
5270
S3AccessGrantsPlugin accessGrantsPlugin =
5371
S3AccessGrantsPlugin.builder().enableFallback(s3agFallbackEnabled).build();
5472
builder.addPlugin(accessGrantsPlugin);
55-
LOG_EXACTLY_ONCE.info("s3ag plugin is added to s3 client with fallback: {}", s3agFallbackEnabled);
56-
} else {
57-
LOG_EXACTLY_ONCE.warn("s3ag plugin is not available.");
73+
LOG_S3AG_PLUGIN_INFO.info("S3 Access Grants plugin is added to s3 client with fallback: {}", s3agFallbackEnabled);
5874
}
5975
}
76+
6077
}

‎hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AccessGrantConfiguration.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,4 +86,5 @@ private Configuration createConfig(boolean isDefault, boolean s3agEnabled) {
8686
assertEquals(builder.plugins().size(), 0);
8787
}
8888
}
89+
8990
}

0 commit comments

Comments
 (0)