HDDS-1393. Convert all OM Bucket related operations to HA model. (#704)

Author: bharatviswa504

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerHAProtocol.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.ozone.om.protocol;
 
+import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
 import org.apache.hadoop.ozone.om.helpers.OmDeleteVolumeResponse;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -166,4 +168,54 @@ void applySetOwner(String oldOwner, VolumeList oldOwnerVolumeList,
    */
   void applyDeleteVolume(String volume, String owner,
       VolumeList newVolumeList) throws IOException;
+
+  /**
+   * Start Create Bucket Transaction.
+   * @param omBucketInfo
+   * @return OmBucketInfo
+   * @throws IOException
+   */
+  OmBucketInfo startCreateBucket(OmBucketInfo omBucketInfo) throws IOException;
+
+  /**
+   * Apply Create Bucket Changes to OM DB.
+   * @param omBucketInfo
+   * @throws IOException
+   */
+  void applyCreateBucket(OmBucketInfo omBucketInfo) throws IOException;
+
+  /**
+   * Start Delete Bucket Transaction.
+   * @param volumeName
+   * @param bucketName
+   * @throws IOException
+   */
+  void startDeleteBucket(String volumeName, String bucketName)
+      throws IOException;
+
+  /**
+   * Apply Delete Bucket changes to OM DB.
+   * @param volumeName
+   * @param bucketName
+   * @throws IOException
+   */
+  void applyDeleteBucket(String volumeName, String bucketName)
+      throws IOException;
+
+  /**
+   * Start SetBucket Property Transaction.
+   * @param omBucketArgs
+   * @return OmBucketInfo
+   * @throws IOException
+   */
+  OmBucketInfo startSetBucketProperty(OmBucketArgs omBucketArgs)
+      throws IOException;
+
+  /**
+   * Apply SetBucket Property changes to OM DB.
+   * @param omBucketInfo
+   * @throws IOException
+   */
+  void applySetBucketProperty(OmBucketInfo omBucketInfo) throws IOException;
+
 }

hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto

@@ -478,7 +478,11 @@ message InfoBucketResponse {
 }
 
 message SetBucketPropertyRequest {
-    required BucketArgs bucketArgs = 1;
+    //TODO: See if we can merge bucketArgs and bucketInfo
+    optional BucketArgs bucketArgs = 1;
+    // This will be set during startTransaction, and used to apply to OM DB
+    // during applyTransaction.
+    optional BucketInfo bucketInfo = 2;
 }
 
 message SetBucketPropertyResponse {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java

@@ -171,11 +171,11 @@ public void testBucketOps() throws IOException {
     Mockito.doNothing().when(mockS3Bm).deleteS3Bucket("random");
     Mockito.doReturn(true).when(mockS3Bm).createOzoneVolumeIfNeeded(null);
 
-    Mockito.doNothing().when(mockBm).createBucket(null);
-    Mockito.doNothing().when(mockBm).createBucket(null);
+    Mockito.doReturn(null).when(mockBm).createBucket(null);
+    Mockito.doReturn(null).when(mockBm).createBucket(null);
     Mockito.doNothing().when(mockBm).deleteBucket(null, null);
     Mockito.doReturn(null).when(mockBm).getBucketInfo(null, null);
-    Mockito.doNothing().when(mockBm).setBucketProperty(null);
+    Mockito.doReturn(null).when(mockBm).setBucketProperty(null);
     Mockito.doReturn(null).when(mockBm).listBuckets(null, null, null, 0);
 
     HddsWhiteboxTestUtils.setInternalState(

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java

@@ -20,11 +20,13 @@
 import org.apache.hadoop.hdds.client.ReplicationFactor;
 import org.apache.hadoop.hdds.client.ReplicationType;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdfs.LogVerificationAppender;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
 import org.apache.hadoop.ozone.OzoneTestUtils;
+import org.apache.hadoop.ozone.client.BucketArgs;
 import org.apache.hadoop.ozone.client.ObjectStore;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneClient;
@@ -38,6 +40,7 @@
 import org.apache.hadoop.ozone.client.OzoneClientFactory;
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.client.VolumeArgs;
+import org.apache.hadoop.util.Time;
 import org.apache.log4j.Logger;
 import org.junit.After;
 import org.junit.Assert;
@@ -125,23 +128,32 @@ public void shutdown() {
     }
   }
 
-  @Test
-  public void testAllVolumeOperations() throws Exception {
+
+  private OzoneVolume createAndCheckVolume(String volumeName)
+      throws Exception {
     String userName = "user" + RandomStringUtils.randomNumeric(5);
     String adminName = "admin" + RandomStringUtils.randomNumeric(5);
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
-
     VolumeArgs createVolumeArgs = VolumeArgs.newBuilder()
         .setOwner(userName)
         .setAdmin(adminName)
         .build();
 
     objectStore.createVolume(volumeName, createVolumeArgs);
-    OzoneVolume retVolumeinfo = objectStore.getVolume(volumeName);
 
-    Assert.assertTrue(retVolumeinfo.getName().equals(volumeName));
-    Assert.assertTrue(retVolumeinfo.getOwner().equals(userName));
-    Assert.assertTrue(retVolumeinfo.getAdmin().equals(adminName));
+    OzoneVolume retVolume = objectStore.getVolume(volumeName);
+
+    Assert.assertTrue(retVolume.getName().equals(volumeName));
+    Assert.assertTrue(retVolume.getOwner().equals(userName));
+    Assert.assertTrue(retVolume.getAdmin().equals(adminName));
+
+    return retVolume;
+  }
+  @Test
+  public void testAllVolumeOperations() throws Exception {
+
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+
+    createAndCheckVolume(volumeName);
 
     objectStore.deleteVolume(volumeName);
 
@@ -152,6 +164,47 @@ public void testAllVolumeOperations() throws Exception {
         () -> objectStore.deleteVolume(volumeName));
   }
 
+
+  @Test
+  public void testAllBucketOperations() throws Exception {
+
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+    String bucketName = "volume" + RandomStringUtils.randomNumeric(5);
+
+    OzoneVolume retVolume = createAndCheckVolume(volumeName);
+
+    BucketArgs bucketArgs =
+        BucketArgs.newBuilder().setStorageType(StorageType.DISK)
+            .setVersioning(true).build();
+
+
+    retVolume.createBucket(bucketName, bucketArgs);
+
+
+    OzoneBucket ozoneBucket = retVolume.getBucket(bucketName);
+
+    Assert.assertEquals(volumeName, ozoneBucket.getVolumeName());
+    Assert.assertEquals(bucketName, ozoneBucket.getName());
+    Assert.assertTrue(ozoneBucket.getVersioning());
+    Assert.assertEquals(StorageType.DISK, ozoneBucket.getStorageType());
+    Assert.assertTrue(ozoneBucket.getCreationTime() <= Time.now());
+
+
+    // Change versioning to false
+    ozoneBucket.setVersioning(false);
+
+    ozoneBucket = retVolume.getBucket(bucketName);
+    Assert.assertFalse(ozoneBucket.getVersioning());
+
+    retVolume.deleteBucket(bucketName);
+
+    OzoneTestUtils.expectOmException(OMException.ResultCodes.BUCKET_NOT_FOUND,
+        () -> retVolume.deleteBucket(bucketName));
+
+
+
+  }
+
   /**
    * Test a client request when all OM nodes are running. The request should
    * succeed.

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManager.java

@@ -30,7 +30,16 @@ public interface BucketManager {
    * Creates a bucket.
    * @param bucketInfo - OmBucketInfo for creating bucket.
    */
-  void createBucket(OmBucketInfo bucketInfo) throws IOException;
+  OmBucketInfo createBucket(OmBucketInfo bucketInfo) throws IOException;
+
+  /**
+   * Apply Create Bucket changes to OM DB.
+   * @param omBucketInfo
+   * @throws IOException
+   */
+  void applyCreateBucket(OmBucketInfo omBucketInfo) throws IOException;
+
+
   /**
    * Returns Bucket Information.
    * @param volumeName - Name of the Volume.
@@ -44,7 +53,14 @@ OmBucketInfo getBucketInfo(String volumeName, String bucketName)
    * @param args - BucketArgs.
    * @throws IOException
    */
-  void setBucketProperty(OmBucketArgs args) throws IOException;
+  OmBucketInfo setBucketProperty(OmBucketArgs args) throws IOException;
+
+  /**
+   * Apply SetBucket Property changes to OM DB.
+   * @param omBucketInfo
+   * @throws IOException
+   */
+  void applySetBucketProperty(OmBucketInfo omBucketInfo) throws IOException;
 
   /**
    * Deletes an existing empty bucket from volume.
@@ -54,6 +70,15 @@ OmBucketInfo getBucketInfo(String volumeName, String bucketName)
    */
   void deleteBucket(String volumeName, String bucketName) throws IOException;
 
+  /**
+   * Apply Delete Bucket changes to OM DB.
+   * @param volumeName
+   * @param bucketName
+   * @throws IOException
+   */
+  void applyDeleteBucket(String volumeName, String bucketName)
+      throws IOException;
+
   /**
    * Returns a list of buckets represented by {@link OmBucketInfo}
    * in the given volume.