HDDS-1065. OM and DN should persist SCM certificate as the trust root. Contributed by Ajay Kumar. (#834)

Author: ajayydv

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java

@@ -178,7 +178,7 @@ public static InetSocketAddress getScmAddressForBlockClients(
    * @return {@link SCMSecurityProtocol}
    * @throws IOException
    */
-  public static SCMSecurityProtocol getScmSecurityClient(
+  public static SCMSecurityProtocolClientSideTranslatorPB getScmSecurityClient(
       OzoneConfiguration conf, InetSocketAddress address) throws IOException {
     RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
         ProtobufRpcEngine.class);

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java

@@ -23,6 +23,7 @@
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto;
 import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCACertificateRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
 import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto;
 import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto.Builder;
 import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
@@ -79,18 +80,8 @@ public void close() throws IOException {
   @Override
   public String getDataNodeCertificate(DatanodeDetailsProto dataNodeDetails,
       String certSignReq) throws IOException {
-    SCMGetDataNodeCertRequestProto.Builder builder =
-        SCMGetDataNodeCertRequestProto
-            .newBuilder()
-            .setCSR(certSignReq)
-            .setDatanodeDetails(dataNodeDetails);
-    try {
-      return rpcProxy
-          .getDataNodeCertificate(NULL_RPC_CONTROLLER, builder.build())
-          .getX509Certificate();
-    } catch (ServiceException e) {
-      throw ProtobufHelper.getRemoteException(e);
-    }
+    return getDataNodeCertificateChain(dataNodeDetails, certSignReq)
+        .getX509Certificate();
   }
 
   /**
@@ -103,13 +94,25 @@ public String getDataNodeCertificate(DatanodeDetailsProto dataNodeDetails,
   @Override
   public String getOMCertificate(OzoneManagerDetailsProto omDetails,
       String certSignReq) throws IOException {
+    return getOMCertChain(omDetails, certSignReq).getX509Certificate();
+  }
+
+  /**
+   * Get SCM signed certificate for OM.
+   *
+   * @param omDetails       - OzoneManager Details.
+   * @param certSignReq     - Certificate signing request.
+   * @return byte[]         - SCM signed certificate.
+   */
+  public SCMGetCertResponseProto getOMCertChain(
+      OzoneManagerDetailsProto omDetails, String certSignReq)
+      throws IOException {
     SCMGetOMCertRequestProto.Builder builder = SCMGetOMCertRequestProto
         .newBuilder()
         .setCSR(certSignReq)
         .setOmDetails(omDetails);
     try {
-      return rpcProxy.getOMCertificate(NULL_RPC_CONTROLLER, builder.build())
-          .getX509Certificate();
+      return rpcProxy.getOMCertificate(NULL_RPC_CONTROLLER, builder.build());
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
     }
@@ -135,6 +138,28 @@ public String getCertificate(String certSerialId) throws IOException {
     }
   }
 
+  /**
+   * Get SCM signed certificate for Datanode.
+   *
+   * @param dnDetails       - Datanode Details.
+   * @param certSignReq     - Certificate signing request.
+   * @return byte[]         - SCM signed certificate.
+   */
+  public SCMGetCertResponseProto getDataNodeCertificateChain(
+      DatanodeDetailsProto dnDetails, String certSignReq)
+      throws IOException {
+    SCMGetDataNodeCertRequestProto.Builder builder =
+        SCMGetDataNodeCertRequestProto.newBuilder()
+            .setCSR(certSignReq)
+            .setDatanodeDetails(dnDetails);
+    try {
+      return rpcProxy.getDataNodeCertificate(NULL_RPC_CONTROLLER,
+          builder.build());
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+  }
+
   /**
    * Get CA certificate.
    *

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java

@@ -61,7 +61,9 @@ public SCMGetCertResponseProto getDataNodeCertificate(
           SCMGetCertResponseProto
               .newBuilder()
               .setResponseCode(ResponseCode.success)
-              .setX509Certificate(certificate);
+              .setX509Certificate(certificate)
+              .setX509CACertificate(impl.getCACertificate());
+
       return builder.build();
     } catch (IOException e) {
       throw new ServiceException(e);
@@ -87,7 +89,8 @@ public SCMGetCertResponseProto getOMCertificate(
           SCMGetCertResponseProto
               .newBuilder()
               .setResponseCode(ResponseCode.success)
-              .setX509Certificate(certificate);
+              .setX509Certificate(certificate)
+              .setX509CACertificate(impl.getCACertificate());
       return builder.build();
     } catch (IOException e) {
       throw new ServiceException(e);

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java

@@ -141,6 +141,19 @@ boolean verifySignature(byte[] data, byte[] signature,
   void storeCertificate(String pemEncodedCert, boolean force)
       throws CertificateException;
 
+  /**
+   * Stores the Certificate  for this client. Don't use this api to add
+   * trusted certificates of others.
+   *
+   * @param pemEncodedCert        - pem encoded X509 Certificate
+   * @param force                 - override any existing file
+   * @param caCert                - Is CA certificate.
+   * @throws CertificateException - on Error.
+   *
+   */
+  void storeCertificate(String pemEncodedCert, boolean force, boolean caCert)
+      throws CertificateException;
+
   /**
    * Stores the trusted chain of certificates.
    *

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java

@@ -80,6 +80,7 @@
 public abstract class DefaultCertificateClient implements CertificateClient {
 
   private static final String CERT_FILE_NAME_FORMAT = "%s.crt";
+  private static final String CA_CERT_PREFIX = "CA-";
   private final Logger logger;
   private final SecurityConfig securityConfig;
   private final KeyCodec keyCodec;
@@ -452,14 +453,30 @@ public X509Certificate queryCertificate(String query) {
    * Stores the Certificate  for this client. Don't use this api to add trusted
    * certificates of others.
    *
-   * @param pemEncodedCert - pem encoded X509 Certificate
-   * @param force - override any existing file
+   * @param pemEncodedCert        - pem encoded X509 Certificate
+   * @param force                 - override any existing file
    * @throws CertificateException - on Error.
    *
    */
   @Override
   public void storeCertificate(String pemEncodedCert, boolean force)
       throws CertificateException {
+    this.storeCertificate(pemEncodedCert, force, false);
+  }
+
+  /**
+   * Stores the Certificate  for this client. Don't use this api to add trusted
+   * certificates of others.
+   *
+   * @param pemEncodedCert        - pem encoded X509 Certificate
+   * @param force                 - override any existing file
+   * @param caCert                - Is CA certificate.
+   * @throws CertificateException - on Error.
+   *
+   */
+  @Override
+  public void storeCertificate(String pemEncodedCert, boolean force,
+      boolean caCert) throws CertificateException {
     CertificateCodec certificateCodec = new CertificateCodec(securityConfig);
     try {
       Path basePath = securityConfig.getCertificateLocation();
@@ -469,6 +486,10 @@ public void storeCertificate(String pemEncodedCert, boolean force)
       String certName = String.format(CERT_FILE_NAME_FORMAT,
           cert.getSerialNumber().toString());
 
+      if(caCert) {
+        certName = CA_CERT_PREFIX + certName;
+      }
+
       certificateCodec.writeCertificate(basePath, certName,
           pemEncodedCert, force);
       certificateMap.putIfAbsent(cert.getSerialNumber().toString(), cert);

hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto

@@ -76,6 +76,7 @@ message SCMGetCertResponseProto {
   }
   required ResponseCode responseCode = 1;
   required string x509Certificate = 2; // Base64 encoded X509 certificate.
+  optional string x509CACertificate = 3; // Base64 encoded CA X509 certificate.
 }
 
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java

@@ -26,7 +26,8 @@
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
+import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.scm.HddsServerUtil;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.security.x509.SecurityConfig;
@@ -271,16 +272,25 @@ private void getSCMSignedCert(OzoneConfiguration config) {
     try {
       PKCS10CertificationRequest csr = getCSR(config);
       // TODO: For SCM CA we should fetch certificate from multiple SCMs.
-      SCMSecurityProtocol secureScmClient =
+      SCMSecurityProtocolClientSideTranslatorPB secureScmClient =
           HddsUtils.getScmSecurityClient(config,
               HddsUtils.getScmAddressForSecurityProtocol(config));
-
-      String pemEncodedCert = secureScmClient.getDataNodeCertificate(
-          datanodeDetails.getProtoBufMessage(), getEncodedString(csr));
-      dnCertClient.storeCertificate(pemEncodedCert, true);
-      datanodeDetails.setCertSerialId(getX509Certificate(pemEncodedCert).
-          getSerialNumber().toString());
-      persistDatanodeDetails(datanodeDetails);
+      SCMGetCertResponseProto response = secureScmClient.
+          getDataNodeCertificateChain(datanodeDetails.getProtoBufMessage(),
+              getEncodedString(csr));
+      // Persist certificates.
+      if(response.hasX509CACertificate()) {
+        String pemEncodedCert = response.getX509Certificate();
+        dnCertClient.storeCertificate(pemEncodedCert, true);
+        dnCertClient.storeCertificate(response.getX509CACertificate(), true,
+            true);
+        datanodeDetails.setCertSerialId(getX509Certificate(pemEncodedCert).
+            getSerialNumber().toString());
+        persistDatanodeDetails(datanodeDetails);
+      } else {
+        throw new RuntimeException("Unable to retrieve datanode certificate " +
+            "chain");
+      }
     } catch (IOException | CertificateException e) {
       LOG.error("Error while storing SCM signed certificate.", e);
       throw new RuntimeException(e);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java

@@ -40,6 +40,7 @@
 import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
 import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
 import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
 import org.apache.hadoop.hdds.security.x509.keys.KeyCodec;
 import org.apache.hadoop.io.Text;
@@ -98,6 +99,7 @@
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_EXPIRED;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
 import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
@@ -780,6 +782,12 @@ public void testSecureOmInitSuccess() throws Exception {
           "SCM signed certificate"));
       X509Certificate certificate = om.getCertificateClient().getCertificate();
       validateCertificate(certificate);
+      String pemEncodedCACert =
+          scm.getSecurityProtocolServer().getCACertificate();
+      X509Certificate caCert = CertificateCodec.getX509Cert(pemEncodedCACert);
+      X509Certificate caCertStored = om.getCertificateClient()
+          .getCertificate(caCert.getSerialNumber().toString());
+      assertEquals(caCert, caCertStored);
     } finally {
       if (scm != null) {
         scm.stop();

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java

@@ -139,7 +139,11 @@ public X509Certificate queryCertificate(String query) {
   @Override
   public void storeCertificate(String cert, boolean force)
       throws CertificateException {
+  }
 
+  @Override
+  public void storeCertificate(String cert, boolean force, boolean caCert)
+      throws CertificateException {
   }
 
   /**

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -43,6 +43,7 @@
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
 import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
 import org.apache.hadoop.hdds.scm.ScmInfo;
@@ -785,8 +786,8 @@ private static ScmBlockLocationProtocol getScmBlockClient(
    * @return {@link SCMSecurityProtocol}
    * @throws IOException
    */
-  private static SCMSecurityProtocol getScmSecurityClient(
-      OzoneConfiguration conf) throws IOException {
+  private static SCMSecurityProtocolClientSideTranslatorPB
+      getScmSecurityClient(OzoneConfiguration conf) throws IOException {
     RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
         ProtobufRpcEngine.class);
     long scmVersion =
@@ -1455,16 +1456,28 @@ private static void getSCMSignedCert(CertificateClient client,
     HddsProtos.OzoneManagerDetailsProto omDetailsProto =
         omDetailsProtoBuilder.build();
     LOG.info("OzoneManager ports added:{}", omDetailsProto.getPortsList());
-    SCMSecurityProtocol secureScmClient = getScmSecurityClient(config);
+    SCMSecurityProtocolClientSideTranslatorPB secureScmClient =
+        getScmSecurityClient(config);
 
-    String pemEncodedCert = secureScmClient.getOMCertificate(omDetailsProto,
-        getEncodedString(csr));
+    SCMGetCertResponseProto response = secureScmClient.
+        getOMCertChain(omDetailsProto, getEncodedString(csr));
+    String pemEncodedCert = response.getX509Certificate();
 
     try {
-      client.storeCertificate(pemEncodedCert, true);
-      // Persist om cert serial id.
-      omStore.setOmCertSerialId(CertificateCodec.
-          getX509Certificate(pemEncodedCert).getSerialNumber().toString());
+
+
+      // Store SCM CA certificate.
+      if(response.hasX509CACertificate()) {
+        String pemEncodedRootCert = response.getX509CACertificate();
+        client.storeCertificate(pemEncodedRootCert, true, true);
+        client.storeCertificate(pemEncodedCert, true);
+        // Persist om cert serial id.
+        omStore.setOmCertSerialId(CertificateCodec.
+            getX509Certificate(pemEncodedCert).getSerialNumber().toString());
+      } else {
+        throw new RuntimeException("Unable to retrieve OM certificate " +
+            "chain");
+      }
     } catch (IOException | CertificateException e) {
       LOG.error("Error while storing SCM signed certificate.", e);
       throw new RuntimeException(e);