This is an automated email from the ASF dual-hosted git repository. ferhui pushed a commit to branch HDFS-17384 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/HDFS-17384 by this push: new 62685b61c19a HDFS-17411. [FGL] Client RPCs involving snapshot support fine-grained lock (#6714) 62685b61c19a is described below commit 62685b61c19a4f061ef3963f069a150c6d885c3d Author: ZanderXu <zande...@apache.org> AuthorDate: Tue Apr 9 09:35:36 2024 +0800 HDFS-17411. [FGL] Client RPCs involving snapshot support fine-grained lock (#6714) --- .../hadoop/hdfs/server/namenode/FSNamesystem.java | 45 ++++++++++++---------- .../namenode/snapshot/SnapshotDeletionGc.java | 5 ++- .../hdfs/server/namenode/TestLeaseManager.java | 3 ++ 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 647110c7601b..ddcf23ce55f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -7161,13 +7161,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, checkOperation(OperationCategory.WRITE); final String operationName = "allowSnapshot"; checkSuperuserPrivilege(operationName, path); - writeLock(); + writeLock(FSNamesystemLockMode.FS); try { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot allow snapshot for " + path); FSDirSnapshotOp.allowSnapshot(dir, snapshotManager, path); } finally { - writeUnlock(operationName, getLockReportInfoSupplier(path)); + writeUnlock(FSNamesystemLockMode.FS, operationName, getLockReportInfoSupplier(path)); } getEditLog().logSync(); logAuditEvent(true, operationName, path, null, null); @@ -7178,13 +7178,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, checkOperation(OperationCategory.WRITE); final String operationName = "disallowSnapshot"; checkSuperuserPrivilege(operationName, path); - writeLock(); + writeLock(FSNamesystemLockMode.FS); try { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot disallow snapshot for " + path); FSDirSnapshotOp.disallowSnapshot(dir, snapshotManager, path); } finally { - writeUnlock(operationName, getLockReportInfoSupplier(path)); + writeUnlock(FSNamesystemLockMode.FS, operationName, getLockReportInfoSupplier(path)); } getEditLog().logSync(); logAuditEvent(true, operationName, path, null, null); @@ -7203,14 +7203,15 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, final FSPermissionChecker pc = getPermissionChecker(); FSPermissionChecker.setOperationType(operationName); try { - writeLock(); + writeLock(FSNamesystemLockMode.FS); try { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot create snapshot for " + snapshotRoot); snapshotPath = FSDirSnapshotOp.createSnapshot(dir, pc, snapshotManager, snapshotRoot, snapshotName, logRetryCache); } finally { - writeUnlock(operationName, getLockReportInfoSupplier(snapshotRoot)); + writeUnlock(FSNamesystemLockMode.FS, operationName, + getLockReportInfoSupplier(snapshotRoot)); } } catch (AccessControlException ace) { logAuditEvent(false, operationName, snapshotRoot); @@ -7240,15 +7241,15 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, final FSPermissionChecker pc = getPermissionChecker(); FSPermissionChecker.setOperationType(operationName); try { - writeLock(); + writeLock(FSNamesystemLockMode.FS); try { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot rename snapshot for " + path); FSDirSnapshotOp.renameSnapshot(dir, pc, snapshotManager, path, snapshotOldName, snapshotNewName, logRetryCache); } finally { - writeUnlock(operationName, getLockReportInfoSupplier(oldSnapshotRoot, - newSnapshotRoot)); + writeUnlock(FSNamesystemLockMode.FS, operationName, + getLockReportInfoSupplier(oldSnapshotRoot, newSnapshotRoot)); } } catch (AccessControlException ace) { logAuditEvent(false, operationName, oldSnapshotRoot, @@ -7275,13 +7276,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, final FSPermissionChecker pc = getPermissionChecker(); FSPermissionChecker.setOperationType(operationName); try { - readLock(); + readLock(FSNamesystemLockMode.FS); try { checkOperation(OperationCategory.READ); status = FSDirSnapshotOp.getSnapshottableDirListing(dir, pc, snapshotManager); } finally { - readUnlock(operationName, getLockReportInfoSupplier(null)); + readUnlock(FSNamesystemLockMode.FS, operationName, getLockReportInfoSupplier(null)); } } catch (AccessControlException ace) { logAuditEvent(false, operationName, null, null, null); @@ -7306,14 +7307,14 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, final FSPermissionChecker pc = getPermissionChecker(); FSPermissionChecker.setOperationType(operationName); try { - readLock(); + readLock(FSNamesystemLockMode.FS); try { checkOperation(OperationCategory.READ); status = FSDirSnapshotOp.getSnapshotListing(dir, pc, snapshotManager, snapshotRoot); success = true; } finally { - readUnlock(operationName, getLockReportInfoSupplier(null)); + readUnlock(FSNamesystemLockMode.FS, operationName, getLockReportInfoSupplier(null)); } } catch (AccessControlException ace) { logAuditEvent(success, operationName, snapshotRoot); @@ -7351,14 +7352,14 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, FSPermissionChecker.setOperationType(operationName); long actualTime = Time.monotonicNow(); try { - readLock(); + readLock(FSNamesystemLockMode.FS); try { checkOperation(OperationCategory.READ); diffs = FSDirSnapshotOp.getSnapshotDiffReport(dir, pc, snapshotManager, path, fromSnapshot, toSnapshot); } finally { - readUnlock(operationName, getLockReportInfoSupplier(fromSnapshotRoot, - toSnapshotRoot)); + readUnlock(FSNamesystemLockMode.FS, operationName, + getLockReportInfoSupplier(fromSnapshotRoot, toSnapshotRoot)); } } catch (AccessControlException ace) { logAuditEvent(false, operationName, fromSnapshotRoot, @@ -7425,7 +7426,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, final FSPermissionChecker pc = getPermissionChecker(); FSPermissionChecker.setOperationType(operationName); try { - readLock(); + readLock(FSNamesystemLockMode.FS); try { checkOperation(OperationCategory.READ); diffs = FSDirSnapshotOp @@ -7433,8 +7434,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, fromSnapshot, toSnapshot, startPath, index, snapshotDiffReportLimit); } finally { - readUnlock(operationName, getLockReportInfoSupplier(fromSnapshotRoot, - toSnapshotRoot)); + readUnlock(FSNamesystemLockMode.FS, operationName, + getLockReportInfoSupplier(fromSnapshotRoot, toSnapshotRoot)); } } catch (AccessControlException ace) { logAuditEvent(false, operationName, fromSnapshotRoot, toSnapshotRoot, @@ -7462,7 +7463,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, FSPermissionChecker.setOperationType(operationName); checkOperation(OperationCategory.WRITE); try { - writeLock(); + // It involves Block while collecting blocks to be deleted. + writeLock(FSNamesystemLockMode.GLOBAL); try { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot delete snapshot for " + snapshotRoot); @@ -7470,7 +7472,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, blocksToBeDeleted = FSDirSnapshotOp.deleteSnapshot(dir, pc, snapshotManager, snapshotRoot, snapshotName, logRetryCache); } finally { - writeUnlock(operationName, getLockReportInfoSupplier(rootPath)); + writeUnlock(FSNamesystemLockMode.GLOBAL, operationName, + getLockReportInfoSupplier(rootPath)); } } catch (AccessControlException ace) { logAuditEvent(false, operationName, rootPath, null, null); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java index c62455c724fe..5b509a7d85f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDeletionGc.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode; import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,14 +73,14 @@ public class SnapshotDeletionGc { private void gcDeletedSnapshot(String name) { final Snapshot.Root deleted; - namesystem.readLock(); + namesystem.readLock(FSNamesystemLockMode.FS); try { deleted = namesystem.getSnapshotManager().chooseDeletedSnapshot(); } catch (Throwable e) { LOG.error("Failed to chooseDeletedSnapshot", e); throw e; } finally { - namesystem.readUnlock("gcDeletedSnapshot"); + namesystem.readUnlock(FSNamesystemLockMode.FS, "gcDeletedSnapshot"); } if (deleted == null) { LOG.trace("{}: no snapshots are marked as deleted.", name); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java index 2759b9e20d7e..6be34537a050 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.namenode.fgl.FSNamesystemLockMode; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.util.Lists; @@ -466,6 +467,8 @@ public class TestLeaseManager { when(fsn.isRunning()).thenReturn(true); when(fsn.hasReadLock()).thenReturn(true); when(fsn.hasWriteLock()).thenReturn(true); + when(fsn.hasReadLock(FSNamesystemLockMode.FS)).thenReturn(true); + when(fsn.hasWriteLock(FSNamesystemLockMode.FS)).thenReturn(true); when(fsn.getFSDirectory()).thenReturn(dir); when(fsn.getMaxLockHoldToReleaseLeaseMs()).thenReturn(maxLockHoldToReleaseLeaseMs); return fsn; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org