Repository: hadoop Updated Branches: refs/heads/branch-3.0 2d1f81997 -> 72e1e1904
HDFS-13831. Make block increment deletion number configurable. Contributed by Ryan Wu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72e1e190 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72e1e190 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72e1e190 Branch: refs/heads/branch-3.0 Commit: 72e1e1904b99365c5abba229958570e0466d1b25 Parents: 2d1f819 Author: Wei-Chiu Chuang <[email protected]> Authored: Tue Aug 28 16:36:30 2018 -0700 Committer: Wei-Chiu Chuang <[email protected]> Committed: Tue Aug 28 16:36:30 2018 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +++++ .../hadoop/hdfs/server/namenode/FSNamesystem.java | 13 ++++++++++--- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 10 +++++----- .../hdfs/server/namenode/TestLargeDirectoryDelete.java | 2 +- 4 files changed, 21 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1e190/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index e1c4a27..7932e68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -360,6 +360,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY = "dfs.namenode.startup.delay.block.deletion.sec"; public static final long DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT = 0L; + /** Block deletion increment. */ + public static final String DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY = + "dfs.namenode.block.deletion.increment"; + public static final int DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT = 1000; + public static final String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES = HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES; public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT = http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1e190/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index a7e9533..c652294 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -419,18 +419,18 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, FSNamesystem.class.getName() + ".audit"); private final int maxCorruptFileBlocksReturn; - static int BLOCK_DELETION_INCREMENT = 1000; private final boolean isPermissionEnabled; private final UserGroupInformation fsOwner; private final String supergroup; private final boolean standbyShouldCheckpoint; + private final int blockDeletionIncrement; /** Interval between each check of lease to release. */ private final long leaseRecheckIntervalMs; /** Maximum time the lock is hold to release lease. */ private final long maxLockHoldToReleaseLeaseMs; - // Batch size for open files response + // Batch size for open files respons private final int maxListOpenFilesResponses; // Scan interval is not configurable. @@ -895,6 +895,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, DFSConfigKeys.DFS_NAMENODE_LIST_OPENFILES_NUM_RESPONSES + " must be a positive integer." ); + + this.blockDeletionIncrement = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY, + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT); + Preconditions.checkArgument(blockDeletionIncrement > 0, + DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY + + " must be a positive integer."); } catch(IOException e) { LOG.error(getClass().getSimpleName() + " initialization failed.", e); close(); @@ -3005,7 +3012,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, while (iter.hasNext()) { writeLock(); try { - for (int i = 0; i < BLOCK_DELETION_INCREMENT && iter.hasNext(); i++) { + for (int i = 0; i < blockDeletionIncrement && iter.hasNext(); i++) { blockManager.removeBlock(iter.next()); } } finally { http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1e190/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 5dd2fb4..95203a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -4814,12 +4814,12 @@ </property> <property> - <name>dfs.reformat.disabled</name> - <value>false</value> + <name>dfs.namenode.block.deletion.increment</name> + <value>1000</value> <description> - Disable reformat of NameNode. If it's value is set to "true" - and metadata directories already exist then attempt to format NameNode - will throw NameNodeFormatException. + The number of block deletion increment. + This setting will control the block increment deletion rate to + ensure that other waiters on the lock can get in. </description> </property> </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/72e1e190/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java index 0882d18..3027c79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java @@ -49,6 +49,7 @@ public class TestLargeDirectoryDelete { static { CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1); CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1); + CONF.setInt(DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY, 1); } /** create a file with a length of <code>filelen</code> */ @@ -137,7 +138,6 @@ public class TestLargeDirectoryDelete { threads[1].start(); final long start = Time.now(); - FSNamesystem.BLOCK_DELETION_INCREMENT = 1; mc.getFileSystem().delete(new Path("/root"), true); // recursive delete final long end = Time.now(); threads[0].endThread(); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
