HDDS-134. SCM CA: OM sends CSR and uses certificate issued by SCM. Contributed by Ajay Kumar.

Author: Ajay Kumar

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java

@@ -109,7 +109,7 @@ boolean verifySignature(byte[] data, byte[] signature,
    *
    * @return CertificateSignRequest.Builder
    */
-  CertificateSignRequest.Builder getCSRBuilder();
+  CertificateSignRequest.Builder getCSRBuilder() throws CertificateException;
 
   /**
    * Get the certificate of well-known entity from SCM.

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DNCertificateClient.java

@@ -19,6 +19,8 @@
 
 package org.apache.hadoop.hdds.security.x509.certificate.client;
 
+import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -30,8 +32,22 @@ public class DNCertificateClient extends DefaultCertificateClient {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(DNCertificateClient.class);
-  DNCertificateClient(SecurityConfig securityConfig, String component) {
-    super(securityConfig, component, LOG);
+  public DNCertificateClient(SecurityConfig securityConfig) {
+    super(securityConfig, LOG);
+  }
+
+  /**
+   * Returns a CSR builder that can be used to creates a Certificate signing
+   * request.
+   *
+   * @return CertificateSignRequest.Builder
+   */
+  @Override
+  public CertificateSignRequest.Builder getCSRBuilder()
+      throws CertificateException {
+    return super.getCSRBuilder()
+        .setDigitalEncryption(false)
+        .setDigitalSignature(false);
   }
 
   public Logger getLogger() {

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java

@@ -21,6 +21,7 @@
 
 import com.google.common.base.Preconditions;
 import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.validator.routines.DomainValidator;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
 import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
 import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
@@ -66,20 +67,16 @@ public abstract class DefaultCertificateClient implements CertificateClient {
 
   private final Logger logger;
   private final SecurityConfig securityConfig;
-  private final String component;
   private final KeyCodec keyCodec;
   private PrivateKey privateKey;
   private PublicKey publicKey;
   private X509Certificate x509Certificate;
 
 
-  DefaultCertificateClient(SecurityConfig securityConfig, String component,
-      Logger log) {
+  DefaultCertificateClient(SecurityConfig securityConfig, Logger log) {
     Objects.requireNonNull(securityConfig);
-    Objects.requireNonNull(component);
-    this.component = component;
     this.securityConfig = securityConfig;
-    keyCodec = new KeyCodec(securityConfig, component);
+    keyCodec = new KeyCodec(securityConfig);
     this.logger = log;
   }
 
@@ -95,15 +92,14 @@ public PrivateKey getPrivateKey() {
       return privateKey;
     }
 
-    Path keyPath = securityConfig.getKeyLocation(component);
+    Path keyPath = securityConfig.getKeyLocation();
     if (OzoneSecurityUtil.checkIfFileExist(keyPath,
         securityConfig.getPrivateKeyFileName())) {
       try {
         privateKey = keyCodec.readPrivateKey();
       } catch (InvalidKeySpecException | NoSuchAlgorithmException
           | IOException e) {
-        getLogger().error("Error while getting private key for {}",
-            component, e);
+        getLogger().error("Error while getting private key.", e);
       }
     }
     return privateKey;
@@ -121,15 +117,14 @@ public PublicKey getPublicKey() {
       return publicKey;
     }
 
-    Path keyPath = securityConfig.getKeyLocation(component);
+    Path keyPath = securityConfig.getKeyLocation();
     if (OzoneSecurityUtil.checkIfFileExist(keyPath,
         securityConfig.getPublicKeyFileName())) {
       try {
         publicKey = keyCodec.readPublicKey();
       } catch (InvalidKeySpecException | NoSuchAlgorithmException
           | IOException e) {
-        getLogger().error("Error while getting private key for {}",
-            component, e);
+        getLogger().error("Error while getting public key.", e);
       }
     }
     return publicKey;
@@ -147,18 +142,18 @@ public X509Certificate getCertificate() {
       return x509Certificate;
     }
 
-    Path certPath = securityConfig.getCertificateLocation(component);
+    Path certPath = securityConfig.getCertificateLocation();
     if (OzoneSecurityUtil.checkIfFileExist(certPath,
         securityConfig.getCertificateFileName())) {
       CertificateCodec certificateCodec =
-          new CertificateCodec(securityConfig, component);
+          new CertificateCodec(securityConfig);
       try {
         X509CertificateHolder x509CertificateHolder =
             certificateCodec.readCertificate();
         x509Certificate =
             CertificateCodec.getX509Certificate(x509CertificateHolder);
       } catch (java.security.cert.CertificateException | IOException e) {
-        getLogger().error("Error reading certificate for {}", component, e);
+        getLogger().error("Error reading certificate.", e);
       }
     }
     return x509Certificate;
@@ -318,8 +313,26 @@ private boolean verifySignature(byte[] data, byte[] signature,
    * @return CertificateSignRequest.Builder
    */
   @Override
-  public CertificateSignRequest.Builder getCSRBuilder() {
-    return new CertificateSignRequest.Builder();
+  public CertificateSignRequest.Builder getCSRBuilder()
+      throws CertificateException {
+    CertificateSignRequest.Builder builder =
+        new CertificateSignRequest.Builder()
+        .setConfiguration(securityConfig.getConfiguration());
+    try {
+      DomainValidator validator = DomainValidator.getInstance();
+      // Add all valid ips.
+      OzoneSecurityUtil.getValidInetsForCurrentHost().forEach(
+          ip -> {
+            builder.addIpAddress(ip.getHostAddress());
+            if(validator.isValid(ip.getCanonicalHostName())) {
+              builder.addDnsName(ip.getCanonicalHostName());
+            }
+          });
+    } catch (IOException e) {
+      throw new CertificateException("Error while adding ip to CSR builder",
+          e, CSR_ERROR);
+    }
+    return builder;
   }
 
   /**
@@ -345,8 +358,7 @@ public X509Certificate queryCertificate(String query) {
   @Override
   public void storeCertificate(X509Certificate certificate)
       throws CertificateException {
-    CertificateCodec certificateCodec = new CertificateCodec(securityConfig,
-        component);
+    CertificateCodec certificateCodec = new CertificateCodec(securityConfig);
     try {
       certificateCodec.writeCertificate(
           new X509CertificateHolder(certificate.getEncoded()));
@@ -595,7 +607,7 @@ protected boolean validateKeyPair(PublicKey pubKey)
    * location.
    * */
   protected void bootstrapClientKeys() throws CertificateException {
-    Path keyPath = securityConfig.getKeyLocation(component);
+    Path keyPath = securityConfig.getKeyLocation();
     if (Files.notExists(keyPath)) {
       try {
         Files.createDirectories(keyPath);
@@ -618,10 +630,9 @@ protected KeyPair createKeyPair() throws CertificateException {
       keyCodec.writePrivateKey(keyPair.getPrivate());
     } catch (NoSuchProviderException | NoSuchAlgorithmException
         | IOException e) {
-      getLogger().error("Error while bootstrapping certificate client for {}",
-          component, e);
-      throw new CertificateException("Error while bootstrapping certificate " +
-          "client for" + component, BOOTSTRAP_ERROR);
+      getLogger().error("Error while bootstrapping certificate client.", e);
+      throw new CertificateException("Error while bootstrapping certificate.",
+          BOOTSTRAP_ERROR);
     }
     return keyPair;
   }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/OMCertificateClient.java

@@ -19,6 +19,7 @@
 
 package org.apache.hadoop.hdds.security.x509.certificate.client;
 
+import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -38,8 +39,8 @@ public class OMCertificateClient extends DefaultCertificateClient {
   private static final Logger LOG =
       LoggerFactory.getLogger(OMCertificateClient.class);
 
-  public OMCertificateClient(SecurityConfig securityConfig, String component) {
-    super(securityConfig, component, LOG);
+  public OMCertificateClient(SecurityConfig securityConfig) {
+    super(securityConfig, LOG);
   }
 
   protected InitResponse handleCase(InitCase init) throws
@@ -96,6 +97,21 @@ protected InitResponse handleCase(InitCase init) throws
     }
   }
 
+  /**
+   * Returns a CSR builder that can be used to creates a Certificate signing
+   * request.
+   *
+   * @return CertificateSignRequest.Builder
+   */
+  @Override
+  public CertificateSignRequest.Builder getCSRBuilder()
+      throws CertificateException {
+    return super.getCSRBuilder()
+        .setDigitalEncryption(true)
+        .setDigitalSignature(true);
+  }
+
+
   public Logger getLogger() {
     return LOG;
   }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateCodec.java

@@ -80,6 +80,16 @@ public CertificateCodec(SecurityConfig config, String component) {
     this.location = securityConfig.getCertificateLocation(component);
   }
 
+  /**
+   * Creates an CertificateCodec.
+   *
+   * @param config - Security Config.
+   */
+  public CertificateCodec(SecurityConfig config) {
+    this.securityConfig = config;
+    this.location = securityConfig.getCertificateLocation();
+  }
+
   /**
    * Creates an CertificateCodec.
    *
@@ -167,6 +177,22 @@ public Path getLocation() {
     return location;
   }
 
+  /**
+   * Gets the X.509 Certificate from PEM encoded String.
+   *
+   * @param pemEncodedString - PEM encoded String.
+   * @return X509Certificate  - Certificate.
+   * @throws CertificateException - Thrown on Failure.
+   * @throws IOException          - Thrown on Failure.
+   */
+  public static X509Certificate getX509Cert(String pemEncodedString)
+      throws CertificateException, IOException {
+    CertificateFactory fact = CertificateFactory.getInstance("X.509");
+    try (InputStream input = IOUtils.toInputStream(pemEncodedString, UTF_8)) {
+      return (X509Certificate) fact.generateCertificate(input);
+    }
+  }
+
   /**
    * Write the Certificate pointed to the location by the configs.
    *

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/utils/CertificateSignRequest.java

@@ -144,6 +144,8 @@ public static class Builder {
     private SecurityConfig config;
     private List<GeneralName> altNames;
     private Boolean ca = false;
+    private boolean digitalSignature;
+    private boolean digitalEncryption;
 
     public CertificateSignRequest.Builder setConfiguration(
         Configuration configuration) {
@@ -171,6 +173,16 @@ public CertificateSignRequest.Builder setScmID(String s) {
       return this;
     }
 
+    public Builder setDigitalSignature(boolean dSign) {
+      this.digitalSignature = dSign;
+      return this;
+    }
+
+    public Builder setDigitalEncryption(boolean dEncryption) {
+      this.digitalEncryption = dEncryption;
+      return this;
+    }
+
     // Support SAN extenion with DNS and RFC822 Name
     // other name type will be added as needed.
     public CertificateSignRequest.Builder addDnsName(String dnsName) {
@@ -200,8 +212,13 @@ public CertificateSignRequest.Builder setCA(Boolean isCA) {
     }
 
     private Extension getKeyUsageExtension() throws IOException {
-      int keyUsageFlag = KeyUsage.digitalSignature | KeyUsage.keyEncipherment
-          | KeyUsage.dataEncipherment | KeyUsage.keyAgreement;
+      int keyUsageFlag = KeyUsage.keyAgreement;
+      if(digitalEncryption){
+        keyUsageFlag |= KeyUsage.keyEncipherment | KeyUsage.dataEncipherment;
+      }
+      if(digitalSignature) {
+        keyUsageFlag |= KeyUsage.digitalSignature;
+      }
 
       if (ca) {
         keyUsageFlag |= KeyUsage.keyCertSign | KeyUsage.cRLSign;

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java

@@ -82,6 +82,7 @@ public enum ErrorCode {
     CRYPTO_SIGN_ERROR,
     CERTIFICATE_ERROR,
     BOOTSTRAP_ERROR,
+    CSR_ERROR,
     CRYPTO_SIGNATURE_VERIFICATION_ERROR
   }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java

@@ -87,6 +87,17 @@ public KeyCodec(SecurityConfig config, String component) {
     this.location = securityConfig.getKeyLocation(component);
   }
 
+  /**
+   * Creates an KeyCodec.
+   *
+   * @param config - Security Config.
+   */
+  public KeyCodec(SecurityConfig config) {
+    this.securityConfig = config;
+    isPosixFileSystem = KeyCodec::isPosix;
+    this.location = securityConfig.getKeyLocation();
+  }
+
   /**
    * Creates an HDDS Key Writer.
    *

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -272,6 +272,8 @@ private OzoneConsts() {
   public static final Metadata.Key<String> USER_METADATA_KEY =
       Metadata.Key.of(OZONE_USER, ASCII_STRING_MARSHALLER);
 
+  public static final String RPC_PORT = "RPC";
+
   // Default OMServiceID for OM Ratis servers to use as RaftGroupId
   public static final String OM_SERVICE_ID_DEFAULT = "omServiceIdDefault";