Repository: hadoop Updated Branches: refs/heads/trunk fdd3bc5f4 -> d805cc27a
HDFS-6773. MiniDFSCluster should skip edit log fsync by default. Contributed by Stephen Chu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d805cc27 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d805cc27 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d805cc27 Branch: refs/heads/trunk Commit: d805cc27a98abbdf14a20ef3127a2c7cb212c765 Parents: fdd3bc5 Author: Colin Patrick Mccabe <[email protected]> Authored: Wed Aug 27 13:33:02 2014 -0700 Committer: Colin Patrick Mccabe <[email protected]> Committed: Wed Aug 27 13:33:02 2014 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 22 +++++++++++++++++--- .../server/datanode/TestFsDatasetCache.java | 1 - .../server/namenode/TestCacheDirectives.java | 1 - 4 files changed, 22 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d805cc27/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 99d5c01..7783243 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -518,6 +518,9 @@ Release 2.6.0 - UNRELEASED HDFS-6690. Deduplicate xattr names in memory. (wang) + HDFS-6773. MiniDFSCluster should skip edit log fsync by default (Stephen + Chu via Colin Patrick McCabe) + BUG FIXES HDFS-6823. dfs.web.authentication.kerberos.principal shows up in logs for http://git-wip-us.apache.org/repos/asf/hadoop/blob/d805cc27/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 98ca316..0e49cfe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -93,6 +93,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetUtil; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; +import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; @@ -172,6 +173,7 @@ public class MiniDFSCluster { private boolean checkDataNodeAddrConfig = false; private boolean checkDataNodeHostConfig = false; private Configuration[] dnConfOverlays; + private boolean skipFsyncForTesting = true; public Builder(Configuration conf) { this.conf = conf; @@ -405,6 +407,15 @@ public class MiniDFSCluster { this.dnConfOverlays = dnConfOverlays; return this; } + + /** + * Default: true + * When true, we skip fsync() calls for speed improvements. + */ + public Builder skipFsyncForTesting(boolean val) { + this.skipFsyncForTesting = val; + return this; + } /** * Construct the actual MiniDFSCluster @@ -472,7 +483,8 @@ public class MiniDFSCluster { builder.checkExitOnShutdown, builder.checkDataNodeAddrConfig, builder.checkDataNodeHostConfig, - builder.dnConfOverlays); + builder.dnConfOverlays, + builder.skipFsyncForTesting); } public class DataNodeProperties { @@ -727,7 +739,8 @@ public class MiniDFSCluster { manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs, operation, null, racks, hosts, null, simulatedCapacities, null, true, false, - MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false, false, null); + MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), + true, false, false, null, true); } private void initMiniDFSCluster( @@ -742,7 +755,8 @@ public class MiniDFSCluster { MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown, boolean checkDataNodeAddrConfig, boolean checkDataNodeHostConfig, - Configuration[] dnConfOverlays) + Configuration[] dnConfOverlays, + boolean skipFsyncForTesting) throws IOException { boolean success = false; try { @@ -782,6 +796,8 @@ public class MiniDFSCluster { + "Standby node since no IPC ports have been specified."); conf.setInt(DFS_HA_LOGROLL_PERIOD_KEY, -1); } + + EditLogFileOutputStream.setShouldSkipFsyncForTesting(skipFsyncForTesting); federation = nnTopology.isFederated(); try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/d805cc27/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java index 5ac13ee..d6e70d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java @@ -108,7 +108,6 @@ public class TestFsDatasetCache { private static CacheManipulator prevCacheManipulator; static { - EditLogFileOutputStream.setShouldSkipFsyncForTesting(false); LogManager.getLogger(FsDatasetCache.class).setLevel(Level.DEBUG); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d805cc27/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index d54b90e..9307692 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -110,7 +110,6 @@ public class TestCacheDirectives { static { NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator()); - EditLogFileOutputStream.setShouldSkipFsyncForTesting(false); } private static final long BLOCK_SIZE = 4096;
