HDFS-13408. MiniDFSCluster to support being built on randomized base directory. Contributed by Xiao Liang
(cherry picked from commit f411de6a79a0a87f03c09366cfe7a7d0726ed932) (cherry picked from commit cf272c5179a9cb4b524016c0fca7c69c9eaa92f1) (cherry picked from commit 956ab12ede390e1eea0a66752e0e6711f47b4b94) (cherry picked from commit 99e82e2c2cf554fa5582da2ec9615fd8b698eecc) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0539b723 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0539b723 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0539b723 Branch: refs/heads/branch-2.9 Commit: 0539b72341e6cacd1e6fb9675182c35730bfaf38 Parents: b78d3c2 Author: Chris Douglas <[email protected]> Authored: Mon Apr 23 11:13:18 2018 -0700 Committer: Chris Douglas <[email protected]> Committed: Mon Apr 23 11:15:41 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 22 +++++++++++++++++++- .../apache/hadoop/hdfs/TestMiniDFSCluster.java | 16 +++++++------- 2 files changed, 28 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0539b723/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 887c635..a643077 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -198,8 +198,28 @@ public class MiniDFSCluster implements AutoCloseable { this.conf = conf; this.storagesPerDatanode = FsDatasetTestUtils.Factory.getFactory(conf).getDefaultNumOfDataDirs(); + if (null == conf.get(HDFS_MINIDFS_BASEDIR)) { + conf.set(HDFS_MINIDFS_BASEDIR, + new File(getBaseDirectory()).getAbsolutePath()); + } } - + + public Builder(Configuration conf, File basedir) { + this.conf = conf; + this.storagesPerDatanode = + FsDatasetTestUtils.Factory.getFactory(conf).getDefaultNumOfDataDirs(); + if (null == basedir) { + throw new IllegalArgumentException( + "MiniDFSCluster base directory cannot be null"); + } + String cdir = conf.get(HDFS_MINIDFS_BASEDIR); + if (cdir != null) { + throw new IllegalArgumentException( + "MiniDFSCluster base directory already defined (" + cdir + ")"); + } + conf.set(HDFS_MINIDFS_BASEDIR, basedir.getAbsolutePath()); + } + /** * Default: 0 */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/0539b723/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java index e1346e9..296ede3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java @@ -200,9 +200,8 @@ public class TestMiniDFSCluster { public void testIsClusterUpAfterShutdown() throws Throwable { Configuration conf = new HdfsConfiguration(); File testDataCluster4 = new File(testDataPath, CLUSTER_4); - String c4Path = testDataCluster4.getAbsolutePath(); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, c4Path); - MiniDFSCluster cluster4 = new MiniDFSCluster.Builder(conf).build(); + MiniDFSCluster cluster4 = + new MiniDFSCluster.Builder(conf, testDataCluster4).build(); try { DistributedFileSystem dfs = cluster4.getFileSystem(); dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); @@ -221,12 +220,11 @@ public class TestMiniDFSCluster { Configuration conf = new HdfsConfiguration(); conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "MYHOST"); File testDataCluster5 = new File(testDataPath, CLUSTER_5); - String c5Path = testDataCluster5.getAbsolutePath(); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, c5Path); - try (MiniDFSCluster cluster5 = new MiniDFSCluster.Builder(conf) - .numDataNodes(1) - .checkDataNodeHostConfig(true) - .build()) { + try (MiniDFSCluster cluster5 = + new MiniDFSCluster.Builder(conf, testDataCluster5) + .numDataNodes(1) + .checkDataNodeHostConfig(true) + .build()) { assertEquals("DataNode hostname config not respected", "MYHOST", cluster5.getDataNodes().get(0).getDatanodeId().getHostName()); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
