HDDS-1551. Implement Bucket Write Requests to use Cache and DoubleBuffer.

hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/cache/PartialTableCache.java

@@ -58,12 +58,16 @@ public PartialTableCache() {
 
   @Override
   public CACHEVALUE get(CACHEKEY cachekey) {
-    return cache.get(cachekey);
+    synchronized (cache) {
+      return cache.get(cachekey);
+    }
   }
 
   @Override
   public void put(CACHEKEY cacheKey, CACHEVALUE value) {
-    cache.put(cacheKey, value);
+    synchronized (cache) {
+      cache.put(cacheKey, value);
+    }
     epochEntries.add(new EpochEntry<>(value.getEpoch(), cacheKey));
   }
 
@@ -83,14 +87,16 @@ private void evictCache(long epoch) {
          iterator.hasNext();) {
       currentEntry = iterator.next();
       CACHEKEY cachekey = currentEntry.getCachekey();
-      CacheValue cacheValue = cache.get(cachekey);
-      if (cacheValue.getEpoch() <= epoch) {
-        cache.remove(cachekey);
-        iterator.remove();
-      } else {
-        // If currentEntry epoch is greater than epoch, we have deleted all
-        // entries less than specified epoch. So, we can break.
-        break;
+      synchronized (cache) {
+        CacheValue cacheValue = cache.get(cachekey);
+        if (cacheValue.getEpoch() <= epoch) {
+          cache.remove(cachekey);
+          iterator.remove();
+        } else {
+          // If currentEntry epoch is greater than epoch, we have deleted all
+          // entries less than specified epoch. So, we can break.
+          break;
+        }
       }
     }
   }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java

@@ -195,6 +195,8 @@ public enum ResultCodes {
 
     FILE_ALREADY_EXISTS,
 
-    NOT_A_FILE
+    NOT_A_FILE,
+
+    PERMISSION_DENIED
   }
 }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerHAProtocol.java

@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.ozone.om.protocol;
 
-import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
-import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
 import org.apache.hadoop.ozone.om.helpers.OmDeleteVolumeResponse;
 import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -168,54 +166,4 @@ void applySetOwner(String oldOwner, VolumeList oldOwnerVolumeList,
    */
   void applyDeleteVolume(String volume, String owner,
       VolumeList newVolumeList) throws IOException;
-
-  /**
-   * Start Create Bucket Transaction.
-   * @param omBucketInfo
-   * @return OmBucketInfo
-   * @throws IOException
-   */
-  OmBucketInfo startCreateBucket(OmBucketInfo omBucketInfo) throws IOException;
-
-  /**
-   * Apply Create Bucket Changes to OM DB.
-   * @param omBucketInfo
-   * @throws IOException
-   */
-  void applyCreateBucket(OmBucketInfo omBucketInfo) throws IOException;
-
-  /**
-   * Start Delete Bucket Transaction.
-   * @param volumeName
-   * @param bucketName
-   * @throws IOException
-   */
-  void startDeleteBucket(String volumeName, String bucketName)
-      throws IOException;
-
-  /**
-   * Apply Delete Bucket changes to OM DB.
-   * @param volumeName
-   * @param bucketName
-   * @throws IOException
-   */
-  void applyDeleteBucket(String volumeName, String bucketName)
-      throws IOException;
-
-  /**
-   * Start SetBucket Property Transaction.
-   * @param omBucketArgs
-   * @return OmBucketInfo
-   * @throws IOException
-   */
-  OmBucketInfo startSetBucketProperty(OmBucketArgs omBucketArgs)
-      throws IOException;
-
-  /**
-   * Apply SetBucket Property changes to OM DB.
-   * @param omBucketInfo
-   * @throws IOException
-   */
-  void applySetBucketProperty(OmBucketInfo omBucketInfo) throws IOException;
-
 }

hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto

@@ -97,6 +97,8 @@ message OMRequest {
 
   required string clientId = 3;
 
+  optional UgiInfo ugiInfo = 4;
+
   optional CreateVolumeRequest              createVolumeRequest            = 11;
   optional SetVolumePropertyRequest         setVolumePropertyRequest       = 12;
   optional CheckVolumeAccessRequest         checkVolumeAccessRequest       = 13;
@@ -258,8 +260,14 @@ enum Status {
     DIRECTORY_NOT_FOUND = 45;
     FILE_ALREADY_EXISTS = 46;
     NOT_A_FILE = 47;
+
+    PERMISSION_DENIED = 48;
 }
 
+message UgiInfo {
+    optional string userName = 1;
+    optional string remoteAddress = 3;
+}
 
 message VolumeInfo {
     required string adminName = 1;
@@ -370,7 +378,7 @@ message BucketInfo {
     repeated OzoneAclInfo acls = 3;
     required bool isVersionEnabled = 4 [default = false];
     required StorageTypeProto storageType = 5 [default = DISK];
-    required uint64 creationTime = 6;
+    optional uint64 creationTime = 6;
     repeated hadoop.hdds.KeyValue metadata = 7;
     optional BucketEncryptionInfoProto beinfo = 8;
 }
@@ -490,11 +498,7 @@ message InfoBucketResponse {
 }
 
 message SetBucketPropertyRequest {
-    //TODO: See if we can merge bucketArgs and bucketInfo
     optional BucketArgs bucketArgs = 1;
-    // This will be set during startTransaction, and used to apply to OM DB
-    // during applyTransaction.
-    optional BucketInfo bucketInfo = 2;
 }
 
 message SetBucketPropertyResponse {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java

@@ -171,11 +171,11 @@ public void testBucketOps() throws IOException {
     Mockito.doNothing().when(mockS3Bm).deleteS3Bucket("random");
     Mockito.doReturn(true).when(mockS3Bm).createOzoneVolumeIfNeeded(null);
 
-    Mockito.doReturn(null).when(mockBm).createBucket(null);
-    Mockito.doReturn(null).when(mockBm).createBucket(null);
+    Mockito.doNothing().when(mockBm).createBucket(null);
+    Mockito.doNothing().when(mockBm).createBucket(null);
     Mockito.doNothing().when(mockBm).deleteBucket(null, null);
     Mockito.doReturn(null).when(mockBm).getBucketInfo(null, null);
-    Mockito.doReturn(null).when(mockBm).setBucketProperty(null);
+    Mockito.doNothing().when(mockBm).setBucketProperty(null);
     Mockito.doReturn(null).when(mockBm).listBuckets(null, null, null, 0);
 
     HddsWhiteboxTestUtils.setInternalState(

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java

@@ -405,7 +405,7 @@ private void createKeyTest(boolean checkSuccess) throws Exception {
         // last running OM as it would fail to get a quorum.
         if (e instanceof RemoteException) {
           GenericTestUtils.assertExceptionContains(
-              "RaftRetryFailureException", e);
+              "NotLeaderException", e);
         }
       } else {
         throw e;
@@ -446,7 +446,7 @@ private void createVolumeTest(boolean checkSuccess) throws Exception {
         // last running OM as it would fail to get a quorum.
         if (e instanceof RemoteException) {
           GenericTestUtils.assertExceptionContains(
-              "RaftRetryFailureException", e);
+              "NotLeaderException", e);
         }
       } else {
         throw e;

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManager.java

@@ -30,14 +30,7 @@ public interface BucketManager {
    * Creates a bucket.
    * @param bucketInfo - OmBucketInfo for creating bucket.
    */
-  OmBucketInfo createBucket(OmBucketInfo bucketInfo) throws IOException;
-
-  /**
-   * Apply Create Bucket changes to OM DB.
-   * @param omBucketInfo
-   * @throws IOException
-   */
-  void applyCreateBucket(OmBucketInfo omBucketInfo) throws IOException;
+  void createBucket(OmBucketInfo bucketInfo) throws IOException;
 
 
   /**
@@ -53,14 +46,7 @@ OmBucketInfo getBucketInfo(String volumeName, String bucketName)
    * @param args - BucketArgs.
    * @throws IOException
    */
-  OmBucketInfo setBucketProperty(OmBucketArgs args) throws IOException;
-
-  /**
-   * Apply SetBucket Property changes to OM DB.
-   * @param omBucketInfo
-   * @throws IOException
-   */
-  void applySetBucketProperty(OmBucketInfo omBucketInfo) throws IOException;
+  void setBucketProperty(OmBucketArgs args) throws IOException;
 
   /**
    * Deletes an existing empty bucket from volume.
@@ -70,15 +56,6 @@ OmBucketInfo getBucketInfo(String volumeName, String bucketName)
    */
   void deleteBucket(String volumeName, String bucketName) throws IOException;
 
-  /**
-   * Apply Delete Bucket changes to OM DB.
-   * @param volumeName
-   * @param bucketName
-   * @throws IOException
-   */
-  void applyDeleteBucket(String volumeName, String bucketName)
-      throws IOException;
-
   /**
    * Returns a list of buckets represented by {@link OmBucketInfo}
    * in the given volume.

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java

@@ -102,7 +102,7 @@ KeyProviderCryptoExtension getKMSProvider() {
    * @param bucketInfo - OmBucketInfo.
    */
   @Override
-  public OmBucketInfo createBucket(OmBucketInfo bucketInfo) throws IOException {
+  public void createBucket(OmBucketInfo bucketInfo) throws IOException {
     Preconditions.checkNotNull(bucketInfo);
     String volumeName = bucketInfo.getVolumeName();
     String bucketName = bucketInfo.getBucketName();
@@ -165,11 +165,8 @@ public OmBucketInfo createBucket(OmBucketInfo bucketInfo) throws IOException {
       }
 
       OmBucketInfo omBucketInfo = omBucketInfoBuilder.build();
-      if (!isRatisEnabled) {
-        commitCreateBucketInfoToDB(omBucketInfo);
-      }
+      commitCreateBucketInfoToDB(omBucketInfo);
       LOG.debug("created bucket: {} in volume: {}", bucketName, volumeName);
-      return omBucketInfo;
     } catch (IOException | DBException ex) {
       if (!(ex instanceof OMException)) {
         LOG.error("Bucket creation failed for bucket:{} in volume:{}",
@@ -182,18 +179,6 @@ public OmBucketInfo createBucket(OmBucketInfo bucketInfo) throws IOException {
     }
   }
 
-
-  public void applyCreateBucket(OmBucketInfo omBucketInfo) throws IOException {
-    Preconditions.checkNotNull(omBucketInfo);
-    try {
-      commitCreateBucketInfoToDB(omBucketInfo);
-    } catch (IOException ex) {
-      LOG.error("Apply CreateBucket Failed for bucket: {}, volume: {}",
-          omBucketInfo.getBucketName(), omBucketInfo.getVolumeName(), ex);
-      throw ex;
-    }
-  }
-
   private void commitCreateBucketInfoToDB(OmBucketInfo omBucketInfo)
       throws IOException {
     String dbBucketKey =
@@ -243,7 +228,7 @@ public OmBucketInfo getBucketInfo(String volumeName, String bucketName)
    * @throws IOException - On Failure.
    */
   @Override
-  public OmBucketInfo setBucketProperty(OmBucketArgs args) throws IOException {
+  public void setBucketProperty(OmBucketArgs args) throws IOException {
     Preconditions.checkNotNull(args);
     String volumeName = args.getVolumeName();
     String bucketName = args.getBucketName();
@@ -296,11 +281,7 @@ public OmBucketInfo setBucketProperty(OmBucketArgs args) throws IOException {
       bucketInfoBuilder.setCreationTime(oldBucketInfo.getCreationTime());
 
       OmBucketInfo omBucketInfo = bucketInfoBuilder.build();
-
-      if (!isRatisEnabled) {
-        commitSetBucketPropertyInfoToDB(omBucketInfo);
-      }
-      return omBucketInfo;
+      commitSetBucketPropertyInfoToDB(omBucketInfo);
     } catch (IOException | DBException ex) {
       if (!(ex instanceof OMException)) {
         LOG.error("Setting bucket property failed for bucket:{} in volume:{}",
@@ -312,18 +293,6 @@ public OmBucketInfo setBucketProperty(OmBucketArgs args) throws IOException {
     }
   }
 
-  public void applySetBucketProperty(OmBucketInfo omBucketInfo)
-      throws IOException {
-    try {
-      commitSetBucketPropertyInfoToDB(omBucketInfo);
-    } catch (IOException ex) {
-      LOG.error("Apply SetBucket property failed for bucket:{} in " +
-              "volume:{}", omBucketInfo.getBucketName(),
-          omBucketInfo.getVolumeName(), ex);
-      throw ex;
-    }
-  }
-
   private void commitSetBucketPropertyInfoToDB(OmBucketInfo omBucketInfo)
       throws IOException {
     commitCreateBucketInfoToDB(omBucketInfo);
@@ -377,10 +346,7 @@ public void deleteBucket(String volumeName, String bucketName)
         throw new OMException("Bucket is not empty",
             OMException.ResultCodes.BUCKET_NOT_EMPTY);
       }
-
-      if (!isRatisEnabled) {
-        commitDeleteBucketInfoToOMDB(bucketKey);
-      }
+      commitDeleteBucketInfoToOMDB(bucketKey);
     } catch (IOException ex) {
       if (!(ex instanceof OMException)) {
         LOG.error("Delete bucket failed for bucket:{} in volume:{}", bucketName,
@@ -392,20 +358,6 @@ public void deleteBucket(String volumeName, String bucketName)
     }
   }
 
-  public void applyDeleteBucket(String volumeName, String bucketName)
-      throws IOException {
-    Preconditions.checkNotNull(volumeName);
-    Preconditions.checkNotNull(bucketName);
-    try {
-      commitDeleteBucketInfoToOMDB(metadataManager.getBucketKey(volumeName,
-          bucketName));
-    } catch (IOException ex) {
-      LOG.error("Apply DeleteBucket Failed for bucket: {}, volume: {}",
-          bucketName, volumeName, ex);
-      throw ex;
-    }
-  }
-
   private void commitDeleteBucketInfoToOMDB(String dbBucketKey)
       throws IOException {
     metadataManager.getBucketTable().delete(dbBucketKey);