Skip to content

Commit 0433ab9

Browse files
bharatviswa504bharatviswa504
committed
HDDS-1551. Implement Bucket Write Requests to use Cache and DoubleBuffer.
1 parent 7221078 commit 0433ab9

36 files changed

+1762
-413
lines changed

‎hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerHAProtocol.java‎

Lines changed: 0 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@
1818

1919
package org.apache.hadoop.ozone.om.protocol;
2020

21-
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
22-
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
2321
import org.apache.hadoop.ozone.om.helpers.OmDeleteVolumeResponse;
2422
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
2523
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -168,54 +166,4 @@ void applySetOwner(String oldOwner, VolumeList oldOwnerVolumeList,
168166
*/
169167
void applyDeleteVolume(String volume, String owner,
170168
VolumeList newVolumeList) throws IOException;
171-
172-
/**
173-
* Start Create Bucket Transaction.
174-
* @param omBucketInfo
175-
* @return OmBucketInfo
176-
* @throws IOException
177-
*/
178-
OmBucketInfo startCreateBucket(OmBucketInfo omBucketInfo) throws IOException;
179-
180-
/**
181-
* Apply Create Bucket Changes to OM DB.
182-
* @param omBucketInfo
183-
* @throws IOException
184-
*/
185-
void applyCreateBucket(OmBucketInfo omBucketInfo) throws IOException;
186-
187-
/**
188-
* Start Delete Bucket Transaction.
189-
* @param volumeName
190-
* @param bucketName
191-
* @throws IOException
192-
*/
193-
void startDeleteBucket(String volumeName, String bucketName)
194-
throws IOException;
195-
196-
/**
197-
* Apply Delete Bucket changes to OM DB.
198-
* @param volumeName
199-
* @param bucketName
200-
* @throws IOException
201-
*/
202-
void applyDeleteBucket(String volumeName, String bucketName)
203-
throws IOException;
204-
205-
/**
206-
* Start SetBucket Property Transaction.
207-
* @param omBucketArgs
208-
* @return OmBucketInfo
209-
* @throws IOException
210-
*/
211-
OmBucketInfo startSetBucketProperty(OmBucketArgs omBucketArgs)
212-
throws IOException;
213-
214-
/**
215-
* Apply SetBucket Property changes to OM DB.
216-
* @param omBucketInfo
217-
* @throws IOException
218-
*/
219-
void applySetBucketProperty(OmBucketInfo omBucketInfo) throws IOException;
220-
221169
}

‎hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -370,7 +370,7 @@ message BucketInfo {
370370
repeated OzoneAclInfo acls = 3;
371371
required bool isVersionEnabled = 4 [default = false];
372372
required StorageTypeProto storageType = 5 [default = DISK];
373-
required uint64 creationTime = 6;
373+
optional uint64 creationTime = 6;
374374
repeated hadoop.hdds.KeyValue metadata = 7;
375375
optional BucketEncryptionInfoProto beinfo = 8;
376376
}
@@ -490,11 +490,7 @@ message InfoBucketResponse {
490490
}
491491

492492
message SetBucketPropertyRequest {
493-
//TODO: See if we can merge bucketArgs and bucketInfo
494493
optional BucketArgs bucketArgs = 1;
495-
// This will be set during startTransaction, and used to apply to OM DB
496-
// during applyTransaction.
497-
optional BucketInfo bucketInfo = 2;
498494
}
499495

500496
message SetBucketPropertyResponse {

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -171,11 +171,11 @@ public void testBucketOps() throws IOException {
171171
Mockito.doNothing().when(mockS3Bm).deleteS3Bucket("random");
172172
Mockito.doReturn(true).when(mockS3Bm).createOzoneVolumeIfNeeded(null);
173173

174-
Mockito.doReturn(null).when(mockBm).createBucket(null);
175-
Mockito.doReturn(null).when(mockBm).createBucket(null);
174+
Mockito.doNothing().when(mockBm).createBucket(null);
175+
Mockito.doNothing().when(mockBm).createBucket(null);
176176
Mockito.doNothing().when(mockBm).deleteBucket(null, null);
177177
Mockito.doReturn(null).when(mockBm).getBucketInfo(null, null);
178-
Mockito.doReturn(null).when(mockBm).setBucketProperty(null);
178+
Mockito.doNothing().when(mockBm).setBucketProperty(null);
179179
Mockito.doReturn(null).when(mockBm).listBuckets(null, null, null, 0);
180180

181181
HddsWhiteboxTestUtils.setInternalState(

‎hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,7 @@ private void createKeyTest(boolean checkSuccess) throws Exception {
405405
// last running OM as it would fail to get a quorum.
406406
if (e instanceof RemoteException) {
407407
GenericTestUtils.assertExceptionContains(
408-
"RaftRetryFailureException", e);
408+
"NotLeaderException", e);
409409
}
410410
} else {
411411
throw e;
@@ -446,7 +446,7 @@ private void createVolumeTest(boolean checkSuccess) throws Exception {
446446
// last running OM as it would fail to get a quorum.
447447
if (e instanceof RemoteException) {
448448
GenericTestUtils.assertExceptionContains(
449-
"RaftRetryFailureException", e);
449+
"NotLeaderException", e);
450450
}
451451
} else {
452452
throw e;

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManager.java‎

Lines changed: 2 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,7 @@ public interface BucketManager {
3030
* Creates a bucket.
3131
* @param bucketInfo - OmBucketInfo for creating bucket.
3232
*/
33-
OmBucketInfo createBucket(OmBucketInfo bucketInfo) throws IOException;
34-
35-
/**
36-
* Apply Create Bucket changes to OM DB.
37-
* @param omBucketInfo
38-
* @throws IOException
39-
*/
40-
void applyCreateBucket(OmBucketInfo omBucketInfo) throws IOException;
33+
void createBucket(OmBucketInfo bucketInfo) throws IOException;
4134

4235

4336
/**
@@ -53,14 +46,7 @@ OmBucketInfo getBucketInfo(String volumeName, String bucketName)
5346
* @param args - BucketArgs.
5447
* @throws IOException
5548
*/
56-
OmBucketInfo setBucketProperty(OmBucketArgs args) throws IOException;
57-
58-
/**
59-
* Apply SetBucket Property changes to OM DB.
60-
* @param omBucketInfo
61-
* @throws IOException
62-
*/
63-
void applySetBucketProperty(OmBucketInfo omBucketInfo) throws IOException;
49+
void setBucketProperty(OmBucketArgs args) throws IOException;
6450

6551
/**
6652
* Deletes an existing empty bucket from volume.
@@ -70,15 +56,6 @@ OmBucketInfo getBucketInfo(String volumeName, String bucketName)
7056
*/
7157
void deleteBucket(String volumeName, String bucketName) throws IOException;
7258

73-
/**
74-
* Apply Delete Bucket changes to OM DB.
75-
* @param volumeName
76-
* @param bucketName
77-
* @throws IOException
78-
*/
79-
void applyDeleteBucket(String volumeName, String bucketName)
80-
throws IOException;
81-
8259
/**
8360
* Returns a list of buckets represented by {@link OmBucketInfo}
8461
* in the given volume.

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java‎

Lines changed: 5 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ KeyProviderCryptoExtension getKMSProvider() {
102102
* @param bucketInfo - OmBucketInfo.
103103
*/
104104
@Override
105-
public OmBucketInfo createBucket(OmBucketInfo bucketInfo) throws IOException {
105+
public void createBucket(OmBucketInfo bucketInfo) throws IOException {
106106
Preconditions.checkNotNull(bucketInfo);
107107
String volumeName = bucketInfo.getVolumeName();
108108
String bucketName = bucketInfo.getBucketName();
@@ -165,11 +165,8 @@ public OmBucketInfo createBucket(OmBucketInfo bucketInfo) throws IOException {
165165
}
166166

167167
OmBucketInfo omBucketInfo = omBucketInfoBuilder.build();
168-
if (!isRatisEnabled) {
169-
commitCreateBucketInfoToDB(omBucketInfo);
170-
}
168+
commitCreateBucketInfoToDB(omBucketInfo);
171169
LOG.debug("created bucket: {} in volume: {}", bucketName, volumeName);
172-
return omBucketInfo;
173170
} catch (IOException | DBException ex) {
174171
if (!(ex instanceof OMException)) {
175172
LOG.error("Bucket creation failed for bucket:{} in volume:{}",
@@ -182,18 +179,6 @@ public OmBucketInfo createBucket(OmBucketInfo bucketInfo) throws IOException {
182179
}
183180
}
184181

185-
186-
public void applyCreateBucket(OmBucketInfo omBucketInfo) throws IOException {
187-
Preconditions.checkNotNull(omBucketInfo);
188-
try {
189-
commitCreateBucketInfoToDB(omBucketInfo);
190-
} catch (IOException ex) {
191-
LOG.error("Apply CreateBucket Failed for bucket: {}, volume: {}",
192-
omBucketInfo.getBucketName(), omBucketInfo.getVolumeName(), ex);
193-
throw ex;
194-
}
195-
}
196-
197182
private void commitCreateBucketInfoToDB(OmBucketInfo omBucketInfo)
198183
throws IOException {
199184
String dbBucketKey =
@@ -243,7 +228,7 @@ public OmBucketInfo getBucketInfo(String volumeName, String bucketName)
243228
* @throws IOException - On Failure.
244229
*/
245230
@Override
246-
public OmBucketInfo setBucketProperty(OmBucketArgs args) throws IOException {
231+
public void setBucketProperty(OmBucketArgs args) throws IOException {
247232
Preconditions.checkNotNull(args);
248233
String volumeName = args.getVolumeName();
249234
String bucketName = args.getBucketName();
@@ -296,11 +281,7 @@ public OmBucketInfo setBucketProperty(OmBucketArgs args) throws IOException {
296281
bucketInfoBuilder.setCreationTime(oldBucketInfo.getCreationTime());
297282

298283
OmBucketInfo omBucketInfo = bucketInfoBuilder.build();
299-
300-
if (!isRatisEnabled) {
301-
commitSetBucketPropertyInfoToDB(omBucketInfo);
302-
}
303-
return omBucketInfo;
284+
commitSetBucketPropertyInfoToDB(omBucketInfo);
304285
} catch (IOException | DBException ex) {
305286
if (!(ex instanceof OMException)) {
306287
LOG.error("Setting bucket property failed for bucket:{} in volume:{}",
@@ -312,18 +293,6 @@ public OmBucketInfo setBucketProperty(OmBucketArgs args) throws IOException {
312293
}
313294
}
314295

315-
public void applySetBucketProperty(OmBucketInfo omBucketInfo)
316-
throws IOException {
317-
try {
318-
commitSetBucketPropertyInfoToDB(omBucketInfo);
319-
} catch (IOException ex) {
320-
LOG.error("Apply SetBucket property failed for bucket:{} in " +
321-
"volume:{}", omBucketInfo.getBucketName(),
322-
omBucketInfo.getVolumeName(), ex);
323-
throw ex;
324-
}
325-
}
326-
327296
private void commitSetBucketPropertyInfoToDB(OmBucketInfo omBucketInfo)
328297
throws IOException {
329298
commitCreateBucketInfoToDB(omBucketInfo);
@@ -377,10 +346,7 @@ public void deleteBucket(String volumeName, String bucketName)
377346
throw new OMException("Bucket is not empty",
378347
OMException.ResultCodes.BUCKET_NOT_EMPTY);
379348
}
380-
381-
if (!isRatisEnabled) {
382-
commitDeleteBucketInfoToOMDB(bucketKey);
383-
}
349+
commitDeleteBucketInfoToOMDB(bucketKey);
384350
} catch (IOException ex) {
385351
if (!(ex instanceof OMException)) {
386352
LOG.error("Delete bucket failed for bucket:{} in volume:{}", bucketName,
@@ -392,20 +358,6 @@ public void deleteBucket(String volumeName, String bucketName)
392358
}
393359
}
394360

395-
public void applyDeleteBucket(String volumeName, String bucketName)
396-
throws IOException {
397-
Preconditions.checkNotNull(volumeName);
398-
Preconditions.checkNotNull(bucketName);
399-
try {
400-
commitDeleteBucketInfoToOMDB(metadataManager.getBucketKey(volumeName,
401-
bucketName));
402-
} catch (IOException ex) {
403-
LOG.error("Apply DeleteBucket Failed for bucket: {}, volume: {}",
404-
bucketName, volumeName, ex);
405-
throw ex;
406-
}
407-
}
408-
409361
private void commitDeleteBucketInfoToOMDB(String dbBucketKey)
410362
throws IOException {
411363
metadataManager.getBucketTable().delete(dbBucketKey);

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java‎

Lines changed: 6 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,7 @@ private OzoneManager(OzoneConfiguration conf) throws IOException,
313313
RPC.setProtocolEngine(configuration, OzoneManagerProtocolPB.class,
314314
ProtobufRpcEngine.class);
315315

316+
metadataManager = new OmMetadataManagerImpl(configuration);
316317
startRatisServer();
317318
startRatisClient();
318319

@@ -325,7 +326,6 @@ private OzoneManager(OzoneConfiguration conf) throws IOException,
325326

326327
secConfig = new SecurityConfig(configuration);
327328

328-
metadataManager = new OmMetadataManagerImpl(configuration);
329329
volumeManager = new VolumeManagerImpl(metadataManager, configuration);
330330

331331
// Create the KMS Key Provider
@@ -1270,7 +1270,7 @@ private RPC.Server getRpcServer(OzoneConfiguration conf) throws IOException {
12701270

12711271
BlockingService omService = newReflectiveBlockingService(
12721272
new OzoneManagerProtocolServerSideTranslatorPB(this, omRatisServer,
1273-
omRatisClient, isRatisEnabled));
1273+
isRatisEnabled));
12741274
return startRpcServer(configuration, omNodeRpcAddr,
12751275
OzoneManagerProtocolPB.class, omService,
12761276
handlerCount);
@@ -1709,67 +1709,6 @@ public void applyDeleteVolume(String volume, String owner,
17091709
volumeManager.applyDeleteVolume(volume, owner, newVolumeList);
17101710
}
17111711

1712-
1713-
@Override
1714-
public OmBucketInfo startCreateBucket(OmBucketInfo omBucketInfo)
1715-
throws IOException {
1716-
Preconditions.checkNotNull(omBucketInfo);
1717-
if(isAclEnabled) {
1718-
checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.CREATE,
1719-
omBucketInfo.getVolumeName(), omBucketInfo.getBucketName(), null);
1720-
}
1721-
1722-
return bucketManager.createBucket(omBucketInfo);
1723-
}
1724-
1725-
@Override
1726-
public void applyCreateBucket(OmBucketInfo omBucketInfo) throws IOException {
1727-
// TODO: Need to add metrics and Audit log for HA requests
1728-
bucketManager.applyCreateBucket(omBucketInfo);
1729-
}
1730-
1731-
1732-
@Override
1733-
public void startDeleteBucket(String volumeName, String bucketName)
1734-
throws IOException {
1735-
// TODO: Need to add metrics and Audit log for HA requests
1736-
if(isAclEnabled) {
1737-
checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.CREATE,
1738-
volumeName, bucketName, null);
1739-
}
1740-
1741-
bucketManager.deleteBucket(volumeName, bucketName);
1742-
}
1743-
1744-
1745-
@Override
1746-
public void applyDeleteBucket(String volumeName, String bucketName)
1747-
throws IOException {
1748-
// TODO: Need to add metrics and Audit log for HA requests
1749-
bucketManager.applyDeleteBucket(volumeName, bucketName);
1750-
}
1751-
1752-
1753-
@Override
1754-
public OmBucketInfo startSetBucketProperty(OmBucketArgs omBucketArgs)
1755-
throws IOException {
1756-
Preconditions.checkNotNull(omBucketArgs);
1757-
// TODO: Need to add metrics and Audit log for HA requests
1758-
if(isAclEnabled) {
1759-
checkAcls(ResourceType.BUCKET, StoreType.OZONE, ACLType.CREATE,
1760-
omBucketArgs.getVolumeName(), omBucketArgs.getBucketName(), null);
1761-
}
1762-
return bucketManager.setBucketProperty(omBucketArgs);
1763-
}
1764-
1765-
1766-
@Override
1767-
public void applySetBucketProperty(OmBucketInfo omBucketInfo)
1768-
throws IOException {
1769-
// TODO: Need to add metrics and Audit log for HA requests
1770-
bucketManager.applySetBucketProperty(omBucketInfo);
1771-
}
1772-
17731712
/**
17741713
* Checks if current caller has acl permissions.
17751714
*
@@ -3037,4 +2976,8 @@ public String getComponent() {
30372976
public OMFailoverProxyProvider getOMFailoverProxyProvider() {
30382977
return null;
30392978
}
2979+
2980+
public OMMetrics getOmMetrics() {
2981+
return metrics;
2982+
}
30402983
}

0 commit comments

Comments
 (0)