Skip to content

Commit 565e848

Browse files
tomscuttomscut
authored
HDFS-16434. Add opname to read/write lock for remaining operations (#3915)
1 parent ffa0eab commit 565e848

20 files changed

+66
-52
lines changed

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -383,7 +383,7 @@ protected void logUpdateMasterKey(DelegationKey key)
383383
namesystem.logUpdateMasterKey(key);
384384
}
385385
} finally {
386-
namesystem.readUnlock();
386+
namesystem.readUnlock("logUpdateMasterKey");
387387
}
388388
} catch (InterruptedException ie) {
389389
// AbstractDelegationTokenManager may crash if an exception is thrown.
@@ -412,7 +412,7 @@ protected void logExpireToken(final DelegationTokenIdentifier dtId)
412412
namesystem.logExpireDelegationToken(dtId);
413413
}
414414
} finally {
415-
namesystem.readUnlock();
415+
namesystem.readUnlock("logExpireToken");
416416
}
417417
} catch (InterruptedException ie) {
418418
// AbstractDelegationTokenManager may crash if an exception is thrown.

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2017,7 +2017,7 @@ int computeBlockReconstructionWork(int blocksToProcess) {
20172017
blocksToReconstruct = neededReconstruction
20182018
.chooseLowRedundancyBlocks(blocksToProcess, reset);
20192019
} finally {
2020-
namesystem.writeUnlock();
2020+
namesystem.writeUnlock("computeBlockReconstructionWork");
20212021
}
20222022
return computeReconstructionWorkForBlocks(blocksToReconstruct);
20232023
}
@@ -2051,7 +2051,7 @@ int computeReconstructionWorkForBlocks(
20512051
}
20522052
}
20532053
} finally {
2054-
namesystem.writeUnlock();
2054+
namesystem.writeUnlock("computeReconstructionWorkForBlocks");
20552055
}
20562056

20572057
// Step 2: choose target nodes for each reconstruction task
@@ -2092,7 +2092,7 @@ int computeReconstructionWorkForBlocks(
20922092
}
20932093
}
20942094
} finally {
2095-
namesystem.writeUnlock();
2095+
namesystem.writeUnlock("computeReconstructionWorkForBlocks");
20962096
}
20972097

20982098
if (blockLog.isDebugEnabled()) {
@@ -2577,7 +2577,7 @@ void processPendingReconstructions() {
25772577
}
25782578
}
25792579
} finally {
2580-
namesystem.writeUnlock();
2580+
namesystem.writeUnlock("processPendingReconstructions");
25812581
}
25822582
/* If we know the target datanodes where the replication timedout,
25832583
* we could invoke decBlocksScheduled() on it. Its ok for now.
@@ -2826,7 +2826,7 @@ public boolean processReport(final DatanodeID nodeID,
28262826
storageInfo.receivedBlockReport();
28272827
} finally {
28282828
endTime = Time.monotonicNow();
2829-
namesystem.writeUnlock();
2829+
namesystem.writeUnlock("processReport");
28302830
}
28312831

28322832
if(blockLog.isDebugEnabled()) {
@@ -2870,7 +2870,7 @@ public void removeBRLeaseIfNeeded(final DatanodeID nodeID,
28702870
context.getTotalRpcs(), Long.toHexString(context.getReportId()));
28712871
}
28722872
} finally {
2873-
namesystem.writeUnlock();
2873+
namesystem.writeUnlock("removeBRLeaseIfNeeded");
28742874
}
28752875
}
28762876

@@ -2908,7 +2908,7 @@ void rescanPostponedMisreplicatedBlocks() {
29082908
postponedMisreplicatedBlocks.addAll(rescannedMisreplicatedBlocks);
29092909
rescannedMisreplicatedBlocks.clear();
29102910
long endSize = postponedMisreplicatedBlocks.size();
2911-
namesystem.writeUnlock();
2911+
namesystem.writeUnlock("rescanPostponedMisreplicatedBlocks");
29122912
LOG.info("Rescan of postponedMisreplicatedBlocks completed in {}" +
29132913
" msecs. {} blocks are left. {} blocks were removed.",
29142914
(Time.monotonicNow() - startTime), endSize, (startSize - endSize));
@@ -3775,7 +3775,7 @@ private void processMisReplicatesAsync() throws InterruptedException {
37753775
break;
37763776
}
37773777
} finally {
3778-
namesystem.writeUnlock();
3778+
namesystem.writeUnlock("processMisReplicatesAsync");
37793779
// Make sure it is out of the write lock for sufficiently long time.
37803780
Thread.sleep(sleepDuration);
37813781
}
@@ -3830,7 +3830,7 @@ public int processMisReplicatedBlocks(List<BlockInfo> blocks) {
38303830
"Re-scanned block {}, result is {}", blk, r);
38313831
}
38323832
} finally {
3833-
namesystem.writeUnlock();
3833+
namesystem.writeUnlock("processMisReplicatedBlocks");
38343834
}
38353835
}
38363836
} catch (InterruptedException ex) {
@@ -4553,7 +4553,7 @@ void processExtraRedundancyBlocksOnInService(
45534553
// testPlacementWithLocalRackNodesDecommissioned, it is not protected by
45544554
// lock, only when called by DatanodeManager.refreshNodes have writeLock
45554555
if (namesystem.hasWriteLock()) {
4556-
namesystem.writeUnlock();
4556+
namesystem.writeUnlock("processExtraRedundancyBlocksOnInService");
45574557
try {
45584558
Thread.sleep(1);
45594559
} catch (InterruptedException e) {
@@ -4685,7 +4685,7 @@ private void updateNeededReconstructions(final BlockInfo block,
46854685
repl.outOfServiceReplicas(), oldExpectedReplicas);
46864686
}
46874687
} finally {
4688-
namesystem.writeUnlock();
4688+
namesystem.writeUnlock("updateNeededReconstructions");
46894689
}
46904690
}
46914691

@@ -4742,7 +4742,7 @@ private int invalidateWorkForOneNode(DatanodeInfo dn) {
47424742
return 0;
47434743
}
47444744
} finally {
4745-
namesystem.writeUnlock();
4745+
namesystem.writeUnlock("invalidateWorkForOneNode");
47464746
}
47474747
blockLog.debug("BLOCK* {}: ask {} to delete {}", getClass().getSimpleName(),
47484748
dn, toInvalidate);
@@ -4974,7 +4974,7 @@ private void remove(long time) {
49744974
}
49754975
}
49764976
} finally {
4977-
namesystem.writeUnlock();
4977+
namesystem.writeUnlock("markedDeleteBlockScrubberThread");
49784978
}
49794979
}
49804980
}
@@ -5092,7 +5092,7 @@ int computeDatanodeWork() {
50925092
this.updateState();
50935093
this.scheduledReplicationBlocksCount = workFound;
50945094
} finally {
5095-
namesystem.writeUnlock();
5095+
namesystem.writeUnlock("computeDatanodeWork");
50965096
}
50975097
workFound += this.computeInvalidateWork(nodesToProcess);
50985098
return workFound;
@@ -5332,7 +5332,7 @@ private void processQueue() {
53325332
action = queue.poll();
53335333
} while (action != null);
53345334
} finally {
5335-
namesystem.writeUnlock();
5335+
namesystem.writeUnlock("processQueue");
53365336
metrics.addBlockOpsBatched(processed - 1);
53375337
}
53385338
} catch (InterruptedException e) {

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -670,7 +670,7 @@ public void run() {
670670
break;
671671
}
672672
} finally {
673-
namesystem.writeUnlock();
673+
namesystem.writeUnlock("leaveSafeMode");
674674
}
675675

676676
try {

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,7 @@ private void rescan() throws InterruptedException {
302302
rescanCachedBlockMap();
303303
blockManager.getDatanodeManager().resetLastCachingDirectiveSentTime();
304304
} finally {
305-
namesystem.writeUnlock();
305+
namesystem.writeUnlock("cacheReplicationMonitorRescan");
306306
}
307307
}
308308

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,7 @@ public void run() {
215215

216216
processPendingNodes();
217217
} finally {
218-
namesystem.writeUnlock();
218+
namesystem.writeUnlock("DatanodeAdminMonitorV2Thread");
219219
}
220220
// After processing the above, various parts of the check() method will
221221
// take and drop the read / write lock as needed. Aside from the
@@ -345,12 +345,12 @@ private void processMaintenanceNodes() {
345345
// which added the node to the cancelled list. Therefore expired
346346
// maintenance nodes do not need to be added to the toRemove list.
347347
dnAdmin.stopMaintenance(dn);
348-
namesystem.writeUnlock();
348+
namesystem.writeUnlock("processMaintenanceNodes");
349349
namesystem.writeLock();
350350
}
351351
}
352352
} finally {
353-
namesystem.writeUnlock();
353+
namesystem.writeUnlock("processMaintenanceNodes");
354354
}
355355
}
356356

@@ -409,7 +409,7 @@ private void processCompletedNodes(List<DatanodeDescriptor> toRemove) {
409409
}
410410
}
411411
} finally {
412-
namesystem.writeUnlock();
412+
namesystem.writeUnlock("processCompletedNodes");
413413
}
414414
}
415415

@@ -531,7 +531,7 @@ private void moveBlocksToPending() {
531531
// replication
532532
if (blocksProcessed >= blocksPerLock) {
533533
blocksProcessed = 0;
534-
namesystem.writeUnlock();
534+
namesystem.writeUnlock("moveBlocksToPending");
535535
namesystem.writeLock();
536536
}
537537
blocksProcessed++;
@@ -553,7 +553,7 @@ private void moveBlocksToPending() {
553553
}
554554
}
555555
} finally {
556-
namesystem.writeUnlock();
556+
namesystem.writeUnlock("moveBlocksToPending");
557557
}
558558
LOG.debug("{} blocks are now pending replication", pendingCount);
559559
}
@@ -637,7 +637,7 @@ private void scanDatanodeStorage(DatanodeDescriptor dn,
637637
try {
638638
storage = dn.getStorageInfos();
639639
} finally {
640-
namesystem.readUnlock();
640+
namesystem.readUnlock("scanDatanodeStorage");
641641
}
642642

643643
for (DatanodeStorageInfo s : storage) {
@@ -667,7 +667,7 @@ private void scanDatanodeStorage(DatanodeDescriptor dn,
667667
numBlocksChecked++;
668668
}
669669
} finally {
670-
namesystem.readUnlock();
670+
namesystem.readUnlock("scanDatanodeStorage");
671671
}
672672
}
673673
}
@@ -722,7 +722,7 @@ private void processPendingReplication() {
722722
suspectBlocks.getOutOfServiceBlockCount());
723723
}
724724
} finally {
725-
namesystem.writeUnlock();
725+
namesystem.writeUnlock("processPendingReplication");
726726
}
727727
}
728728

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ public void run() {
158158
LOG.warn("DatanodeAdminMonitor caught exception when processing node.",
159159
e);
160160
} finally {
161-
namesystem.writeUnlock();
161+
namesystem.writeUnlock("DatanodeAdminMonitorThread");
162162
}
163163
if (numBlocksChecked + numNodesChecked > 0) {
164164
LOG.info("Checked {} blocks and {} nodes this tick. {} nodes are now " +
@@ -373,7 +373,7 @@ private void processBlocksInternal(
373373
// lock.
374374
// Yielding is required in case of block number is greater than the
375375
// configured per-iteration-limit.
376-
namesystem.writeUnlock();
376+
namesystem.writeUnlock("processBlocksInternal");
377377
try {
378378
LOG.debug("Yielded lock during decommission/maintenance check");
379379
Thread.sleep(0, 500);

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -852,7 +852,7 @@ public void removeDatanode(final DatanodeID node)
852852
+ node + " does not exist");
853853
}
854854
} finally {
855-
namesystem.writeUnlock();
855+
namesystem.writeUnlock("removeDatanode");
856856
}
857857
}
858858

@@ -1296,7 +1296,7 @@ public void refreshNodes(final Configuration conf) throws IOException {
12961296
refreshDatanodes();
12971297
countSoftwareVersions();
12981298
} finally {
1299-
namesystem.writeUnlock();
1299+
namesystem.writeUnlock("refreshNodes");
13001300
}
13011301
}
13021302

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -503,7 +503,7 @@ void heartbeatCheck() {
503503
try {
504504
dm.removeDeadDatanode(dead, !dead.isMaintenance());
505505
} finally {
506-
namesystem.writeUnlock();
506+
namesystem.writeUnlock("removeDeadDatanode");
507507
}
508508
}
509509
if (failedStorage != null) {
@@ -512,7 +512,7 @@ void heartbeatCheck() {
512512
try {
513513
blockManager.removeBlocksAssociatedTo(failedStorage);
514514
} finally {
515-
namesystem.writeUnlock();
515+
namesystem.writeUnlock("removeBlocksAssociatedTo");
516516
}
517517
}
518518
}

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ private synchronized void applyEdits(long firstTxId, int numTxns, byte[] data)
222222
try {
223223
getNamesystem().dir.updateCountForQuota();
224224
} finally {
225-
getNamesystem().writeUnlock();
225+
getNamesystem().writeUnlock("applyEdits");
226226
}
227227
} finally {
228228
backupInputStream.clear();

‎hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ void doCheckpoint() throws IOException {
250250
sig.mostRecentCheckpointTxId);
251251
bnImage.reloadFromImageFile(file, backupNode.getNamesystem());
252252
} finally {
253-
backupNode.namesystem.writeUnlock();
253+
backupNode.namesystem.writeUnlock("doCheckpointByBackupNode");
254254
}
255255
}
256256
rollForwardByApplyingLogs(manifest, bnImage, backupNode.getNamesystem());

0 commit comments

Comments
 (0)