Merge branch 'apache:trunk' into YARN-11610

Author: slfan1989

BUILDING.txt

@@ -319,22 +319,28 @@ Controlling the redistribution of the protobuf-2.5 dependency
     the Hadoop codebase; alongside the move to Protobuf 3.x a private successor
     class, org.apache.hadoop.ipc.internal.ShadedProtobufHelper is now used.
 
-    The hadoop-common JAR still declares a dependency on protobuf-2.5, but this
-    is likely to change in the future. The maven scope of the dependency can be
-    set with the common.protobuf2.scope option.
-    It can be set to "provided" in a build:
-       -Dcommon.protobuf2.scope=provided
-    If this is done then protobuf-2.5.0.jar will no longer be exported as a dependency,
-    and will then be omitted from the share/hadoop/common/lib/ directory of
-    any Hadoop distribution built. Any application declaring a dependency on hadoop-commmon
-    will no longer get the dependency; if they need it then they must explicitly declare it:
+    The hadoop-common module no longer exports its compile-time dependency on
+    protobuf-2.5. Hadoop distributions no longer include it.
+    Any application declaring a dependency on hadoop-commmon will no longer get
+    the artifact added to their classpath.
+    If is still required, then they must explicitly declare it:
 
       <dependency>
         <groupId>com.google.protobuf</groupId>
         <artifactId>protobuf-java</artifactId>
         <version>2.5.0</version>
       </dependency>
 
+    In Hadoop builds the scope of the dependency can be set with the
+    option "common.protobuf2.scope".
+    This can be upgraded from "provided" to "compile" on the maven command line:
+
+           -Dcommon.protobuf2.scope=compile
+
+    If this is done then protobuf-2.5.0.jar will again be exported as a
+    hadoop-common dependency, and included in the share/hadoop/common/lib/
+    directory of any Hadoop distribution built.
+
 ----------------------------------------------------------------------------------
 Building components separately
 

LICENSE-binary

@@ -392,7 +392,6 @@ hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
 leveldb v1.13
 
-com.google.protobuf:protobuf-java:2.5.0
 com.google.protobuf:protobuf-java:3.6.1
 com.google.re2j:re2j:1.1
 com.jcraft:jsch:0.1.54

hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java

@@ -16,25 +16,13 @@
 import org.apache.hadoop.classification.VisibleForTesting;
 import java.nio.ByteBuffer;
 import java.security.SecureRandom;
-import java.util.Collections;
-import java.util.List;
 import java.util.Properties;
 import java.util.Random;
-import javax.security.auth.login.Configuration;
 import javax.servlet.ServletContext;
-import org.apache.curator.RetryPolicy;
 import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.framework.api.ACLProvider;
-import org.apache.curator.framework.imps.DefaultACLProvider;
-import org.apache.curator.retry.ExponentialBackoffRetry;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.ZooDefs.Perms;
-import org.apache.zookeeper.client.ZKClientConfig;
-import org.apache.zookeeper.data.ACL;
-import org.apache.zookeeper.data.Id;
 import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -92,6 +80,16 @@ public class ZKSignerSecretProvider extends RolloverSignerSecretProvider {
   public static final String ZOOKEEPER_KERBEROS_PRINCIPAL =
           CONFIG_PREFIX + "kerberos.principal";
 
+  public static final String ZOOKEEPER_SSL_ENABLED = CONFIG_PREFIX + "ssl.enabled";
+  public static final String ZOOKEEPER_SSL_KEYSTORE_LOCATION =
+      CONFIG_PREFIX + "ssl.keystore.location";
+  public static final String ZOOKEEPER_SSL_KEYSTORE_PASSWORD =
+      CONFIG_PREFIX + "ssl.keystore.password";
+  public static final String ZOOKEEPER_SSL_TRUSTSTORE_LOCATION =
+      CONFIG_PREFIX + "ssl.truststore.location";
+  public static final String ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD =
+      CONFIG_PREFIX + "ssl.truststore.password";
+
   /**
    * Constant for the property that specifies whether or not the Curator client
    * should disconnect from ZooKeeper on shutdown.  The default is "true".  Only
@@ -350,80 +348,33 @@ protected byte[] generateRandomSecret() {
    * This method creates the Curator client and connects to ZooKeeper.
    * @param config configuration properties
    * @return A Curator client
-   * @throws Exception thrown if an error occurred
    */
-  protected CuratorFramework createCuratorClient(Properties config)
-          throws Exception {
-    String connectionString = config.getProperty(
-            ZOOKEEPER_CONNECTION_STRING, "localhost:2181");
-
-    RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
-    ACLProvider aclProvider;
+  protected CuratorFramework createCuratorClient(Properties config) {
+    String connectionString = config.getProperty(ZOOKEEPER_CONNECTION_STRING, "localhost:2181");
     String authType = config.getProperty(ZOOKEEPER_AUTH_TYPE, "none");
-    if (authType.equals("sasl")) {
-      LOG.info("Connecting to ZooKeeper with SASL/Kerberos"
-              + "and using 'sasl' ACLs");
-      String principal = setJaasConfiguration(config);
-      System.setProperty(ZKClientConfig.LOGIN_CONTEXT_NAME_KEY,
-              JAAS_LOGIN_ENTRY_NAME);
-      System.setProperty("zookeeper.authProvider.1",
-              "org.apache.zookeeper.server.auth.SASLAuthenticationProvider");
-      aclProvider = new SASLOwnerACLProvider(principal);
-    } else {  // "none"
-      LOG.info("Connecting to ZooKeeper without authentication");
-      aclProvider = new DefaultACLProvider();     // open to everyone
-    }
-    CuratorFramework cf = CuratorFrameworkFactory.builder()
-            .connectString(connectionString)
-            .retryPolicy(retryPolicy)
-            .aclProvider(aclProvider)
-            .build();
-    cf.start();
-    return cf;
-  }
-
-  private String setJaasConfiguration(Properties config) throws Exception {
-    String keytabFile = config.getProperty(ZOOKEEPER_KERBEROS_KEYTAB).trim();
-    if (keytabFile == null || keytabFile.length() == 0) {
-      throw new IllegalArgumentException(ZOOKEEPER_KERBEROS_KEYTAB
-              + " must be specified");
-    }
-    String principal = config.getProperty(ZOOKEEPER_KERBEROS_PRINCIPAL)
-            .trim();
-    if (principal == null || principal.length() == 0) {
-      throw new IllegalArgumentException(ZOOKEEPER_KERBEROS_PRINCIPAL
-              + " must be specified");
-    }
+    String keytab = config.getProperty(ZOOKEEPER_KERBEROS_KEYTAB, "").trim();
+    String principal = config.getProperty(ZOOKEEPER_KERBEROS_PRINCIPAL, "").trim();
 
-    // This is equivalent to writing a jaas.conf file and setting the system
-    // property, "java.security.auth.login.config", to point to it
-    JaasConfiguration jConf =
-            new JaasConfiguration(JAAS_LOGIN_ENTRY_NAME, principal, keytabFile);
-    Configuration.setConfiguration(jConf);
-    return principal.split("[/@]")[0];
-  }
+    boolean sslEnabled = Boolean.parseBoolean(config.getProperty(ZOOKEEPER_SSL_ENABLED, "false"));
+    String keystoreLocation = config.getProperty(ZOOKEEPER_SSL_KEYSTORE_LOCATION, "");
+    String keystorePassword = config.getProperty(ZOOKEEPER_SSL_KEYSTORE_PASSWORD, "");
+    String truststoreLocation = config.getProperty(ZOOKEEPER_SSL_TRUSTSTORE_LOCATION, "");
+    String truststorePassword = config.getProperty(ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD, "");
 
-  /**
-   * Simple implementation of an {@link ACLProvider} that simply returns an ACL
-   * that gives all permissions only to a single principal.
-   */
-  private static class SASLOwnerACLProvider implements ACLProvider {
-
-    private final List<ACL> saslACL;
-
-    private SASLOwnerACLProvider(String principal) {
-      this.saslACL = Collections.singletonList(
-              new ACL(Perms.ALL, new Id("sasl", principal)));
-    }
-
-    @Override
-    public List<ACL> getDefaultAcl() {
-      return saslACL;
-    }
-
-    @Override
-    public List<ACL> getAclForPath(String path) {
-      return saslACL;
-    }
+    CuratorFramework zkClient =
+        ZookeeperClient.configure()
+            .withConnectionString(connectionString)
+            .withAuthType(authType)
+            .withKeytab(keytab)
+            .withPrincipal(principal)
+            .withJaasLoginEntryName(JAAS_LOGIN_ENTRY_NAME)
+            .enableSSL(sslEnabled)
+            .withKeystore(keystoreLocation)
+            .withKeystorePassword(keystorePassword)
+            .withTruststore(truststoreLocation)
+            .withTruststorePassword(truststorePassword)
+            .create();
+    zkClient.start();
+    return zkClient;
   }
 }