HDFS-10220. A large number of expired leases can make namenode unresponsive and cause failover (Nicolas Fraison via raviprak)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae047655 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae047655 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae047655 Branch: refs/heads/HDFS-7240 Commit: ae047655f4355288406cd5396fb4e3ea7c307b14 Parents: 0af96a1 Author: Ravi Prakash <ravip...@altiscale.com> Authored: Wed Jun 8 13:44:22 2016 -0700 Committer: Ravi Prakash <ravip...@altiscale.com> Committed: Wed Jun 8 13:44:22 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 10 +++++ .../hdfs/server/common/HdfsServerConstants.java | 1 - .../hdfs/server/namenode/FSNamesystem.java | 42 ++++++++++++++++---- .../hdfs/server/namenode/LeaseManager.java | 21 ++++++++-- .../src/main/resources/hdfs-default.xml | 18 +++++++++ .../hdfs/server/namenode/TestLeaseManager.java | 24 ++++++----- 6 files changed, 94 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae047655/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 19e1791..f18a6c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -397,6 +397,16 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_NAMENODE_MAX_XATTR_SIZE_DEFAULT = 16384; public static final int DFS_NAMENODE_MAX_XATTR_SIZE_HARD_LIMIT = 32768; + public static final String DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_KEY = + "dfs.namenode.lease-recheck-interval-ms"; + public static final long DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_DEFAULT = + 2000; + public static final String + DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_KEY = + "dfs.namenode.max-lock-hold-to-release-lease-ms"; + public static final long + DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT = 25; + public static final String DFS_UPGRADE_DOMAIN_FACTOR = "dfs.namenode.upgrade.domain.factor"; public static final int DFS_UPGRADE_DOMAIN_FACTOR_DEFAULT = DFS_REPLICATION_DEFAULT; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae047655/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index b2dda3c..3798394 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -361,7 +361,6 @@ public interface HdfsServerConstants { } String NAMENODE_LEASE_HOLDER = "HDFS_NameNode"; - long NAMENODE_LEASE_RECHECK_INTERVAL = 2000; String CRYPTO_XATTR_ENCRYPTION_ZONE = "raw.hdfs.crypto.encryption.zone"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae047655/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index c9f2487..915ae97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -76,6 +76,10 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RETRY_CACHE_EXPI import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RETRY_CACHE_HEAP_PERCENT_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RETRY_CACHE_HEAP_PERCENT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT; @@ -385,7 +389,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, private final UserGroupInformation fsOwner; private final String supergroup; private final boolean standbyShouldCheckpoint; - + + /** Interval between each check of lease to release. */ + private final long leaseRecheckIntervalMs; + /** Maximum time the lock is hold to release lease. */ + private final long maxLockHoldToReleaseLeaseMs; + // Scan interval is not configurable. private static final long DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL = TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS); @@ -803,6 +812,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, DFSConfigKeys.DFS_NAMENODE_EDEKCACHELOADER_INTERVAL_MS_KEY, DFSConfigKeys.DFS_NAMENODE_EDEKCACHELOADER_INTERVAL_MS_DEFAULT); + this.leaseRecheckIntervalMs = conf.getLong( + DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_KEY, + DFS_NAMENODE_LEASE_RECHECK_INTERVAL_MS_DEFAULT); + this.maxLockHoldToReleaseLeaseMs = conf.getLong( + DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_KEY, + DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT); + // For testing purposes, allow the DT secret manager to be started regardless // of whether security is enabled. alwaysUseDelegationTokensForTests = conf.getBoolean( @@ -847,6 +863,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, return retryCache; } + @VisibleForTesting + public long getLeaseRecheckIntervalMs() { + return leaseRecheckIntervalMs; + } + + @VisibleForTesting + public long getMaxLockHoldToReleaseLeaseMs() { + return maxLockHoldToReleaseLeaseMs; + } + void lockRetryCache() { if (retryCache != null) { retryCache.lock(); @@ -3116,9 +3142,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, if(nrCompleteBlocks == nrBlocks) { finalizeINodeFileUnderConstruction(src, pendingFile, iip.getLatestSnapshotId(), false); - NameNode.stateChangeLog.warn("BLOCK*" - + " internalReleaseLease: All existing blocks are COMPLETE," - + " lease removed, file closed."); + NameNode.stateChangeLog.warn("BLOCK*" + + " internalReleaseLease: All existing blocks are COMPLETE," + + " lease removed, file " + src + " closed."); return true; // closed! } @@ -3155,9 +3181,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, blockManager.hasMinStorage(lastBlock)) { finalizeINodeFileUnderConstruction(src, pendingFile, iip.getLatestSnapshotId(), false); - NameNode.stateChangeLog.warn("BLOCK*" - + " internalReleaseLease: Committed blocks are minimally replicated," - + " lease removed, file closed."); + NameNode.stateChangeLog.warn("BLOCK*" + + " internalReleaseLease: Committed blocks are minimally" + + " replicated, lease removed, file" + src + " closed."); return true; // closed! } // Cannot close file right now, since some blocks @@ -3200,7 +3226,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, finalizeINodeFileUnderConstruction(src, pendingFile, iip.getLatestSnapshotId(), false); NameNode.stateChangeLog.warn("BLOCK* internalReleaseLease: " - + "Removed empty last block and closed file."); + + "Removed empty last block and closed file " + src); return true; } // start recovery of the last block for this file http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae047655/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java index e97aa53..06f6586 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java @@ -336,7 +336,7 @@ public class LeaseManager { } } - Thread.sleep(HdfsServerConstants.NAMENODE_LEASE_RECHECK_INTERVAL); + Thread.sleep(fsnamesystem.getLeaseRecheckIntervalMs()); } catch(InterruptedException ie) { if (LOG.isDebugEnabled()) { LOG.debug(name + " is interrupted", ie); @@ -356,8 +356,11 @@ public class LeaseManager { boolean needSync = false; assert fsnamesystem.hasWriteLock(); - while(!sortedLeases.isEmpty() && sortedLeases.peek().expiredHardLimit()) { - Lease leaseToCheck = sortedLeases.poll(); + long start = monotonicNow(); + + while(!sortedLeases.isEmpty() && sortedLeases.peek().expiredHardLimit() + && !isMaxLockHoldToReleaseLease(start)) { + Lease leaseToCheck = sortedLeases.peek(); LOG.info(leaseToCheck + " has expired hard limit"); final List<Long> removing = new ArrayList<>(); @@ -397,6 +400,11 @@ public class LeaseManager { + leaseToCheck, e); removing.add(id); } + if (isMaxLockHoldToReleaseLease(start)) { + LOG.debug("Breaking out of checkLeases after " + + fsnamesystem.getMaxLockHoldToReleaseLeaseMs() + "ms."); + break; + } } for(Long id : removing) { @@ -407,6 +415,13 @@ public class LeaseManager { return needSync; } + + /** @return true if max lock hold is reached */ + private boolean isMaxLockHoldToReleaseLease(long start) { + return monotonicNow() - start > + fsnamesystem.getMaxLockHoldToReleaseLeaseMs(); + } + @Override public synchronized String toString() { return getClass().getSimpleName() + "= {" http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae047655/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 79f7911..fc2f942 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2590,6 +2590,24 @@ </property> <property> + <name>dfs.namenode.lease-recheck-interval-ms</name> + <value>2000</value> + <description>During the release of lease a lock is hold that make any + operations on the namenode stuck. In order to not block them during + a too long duration we stop releasing lease after this max lock limit. + </description> +</property> + +<property> + <name>dfs.namenode.max-lock-hold-to-release-lease-ms</name> + <value>25</value> + <description>During the release of lease a lock is hold that make any + operations on the namenode stuck. In order to not block them during + a too long duration we stop releasing lease after this max lock limit. + </description> +</property> + +<property> <name>dfs.namenode.startup.delay.block.deletion.sec</name> <value>0</value> <description>The delay in seconds at which we will pause the blocks deletion http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae047655/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java index 3bb7bb7..f823745 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -39,6 +40,8 @@ public class TestLeaseManager { @Rule public Timeout timeout = new Timeout(300000); + public static long maxLockHoldToReleaseLeaseMs = 100; + @Test public void testRemoveLeases() throws Exception { FSNamesystem fsn = mock(FSNamesystem.class); @@ -57,28 +60,28 @@ public class TestLeaseManager { assertEquals(0, lm.getINodeIdWithLeases().size()); } - /** Check that even if LeaseManager.checkLease is not able to relinquish - * leases, the Namenode does't enter an infinite loop while holding the FSN - * write lock and thus become unresponsive + /** Check that LeaseManager.checkLease release some leases */ @Test - public void testCheckLeaseNotInfiniteLoop() { + public void testCheckLease() { LeaseManager lm = new LeaseManager(makeMockFsNameSystem()); + long numLease = 100; + //Make sure the leases we are going to add exceed the hard limit lm.setLeasePeriod(0, 0); - //Add some leases to the LeaseManager - lm.addLease("holder1", INodeId.ROOT_INODE_ID + 1); - lm.addLease("holder2", INodeId.ROOT_INODE_ID + 2); - lm.addLease("holder3", INodeId.ROOT_INODE_ID + 3); - assertEquals(lm.countLease(), 3); + for (long i = 0; i <= numLease - 1; i++) { + //Add some leases to the LeaseManager + lm.addLease("holder"+i, INodeId.ROOT_INODE_ID + i); + } + assertEquals(numLease, lm.countLease()); //Initiate a call to checkLease. This should exit within the test timeout lm.checkLeases(); + assertTrue(lm.countLease() < numLease); } - @Test public void testCountPath() { LeaseManager lm = new LeaseManager(makeMockFsNameSystem()); @@ -112,6 +115,7 @@ public class TestLeaseManager { when(fsn.isRunning()).thenReturn(true); when(fsn.hasWriteLock()).thenReturn(true); when(fsn.getFSDirectory()).thenReturn(dir); + when(fsn.getMaxLockHoldToReleaseLeaseMs()).thenReturn(maxLockHoldToReleaseLeaseMs); return fsn; } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
