Repository: hadoop Updated Branches: refs/heads/branch-2 b01eaa3b4 -> b0b97dfbe refs/heads/trunk caa711b66 -> 39581e3be
HDFS-9139. Enable parallel JUnit tests for HDFS Pre-commit (Contributed by Chris Nauroth and Vinayakumar B) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39581e3b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39581e3b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39581e3b Branch: refs/heads/trunk Commit: 39581e3be2aaeb1eeb7fb98b6bdecd8d4e3c7269 Parents: caa711b Author: Vinayakumar B <[email protected]> Authored: Tue Oct 13 19:00:08 2015 +0530 Committer: Vinayakumar B <[email protected]> Committed: Tue Oct 13 19:00:08 2015 +0530 ---------------------------------------------------------------------- dev-support/test-patch.sh | 10 +----- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ hadoop-hdfs-project/hadoop-hdfs/pom.xml | 35 ++++++++++++++++---- .../TestSWebHdfsFileContextMainOperations.java | 11 +++--- .../sasl/SaslDataTransferTestCase.java | 6 ++++ .../hdfs/qjournal/TestSecureNNWithQJM.java | 6 ++++ .../hdfs/server/datanode/TestBlockScanner.java | 10 ++---- .../server/datanode/TestDataNodeMetrics.java | 5 +-- .../hdfs/server/namenode/TestFSNamesystem.java | 1 + .../server/namenode/TestNameNodeHttpServer.java | 4 +++ .../TestNameNodeRespectsBindHostKeys.java | 4 +++ .../server/namenode/ha/TestEditLogTailer.java | 24 +++++++++----- .../hdfs/server/namenode/ha/TestHAAppend.java | 2 +- .../namenode/ha/TestSeveralNameNodes.java | 4 +-- .../hadoop/hdfs/web/TestHttpsFileSystem.java | 4 +++ .../hadoop/hdfs/web/TestWebHdfsTokens.java | 6 +++- ...ClientCredentialTimeBasedTokenRefresher.java | 3 +- 17 files changed, 95 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/dev-support/test-patch.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index 23b2272..3c52912 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -2261,16 +2261,8 @@ function check_unittests test_logfile=${PATCH_DIR}/testrun_${module_suffix}.txt echo " Running tests in ${module_suffix}" - # Temporary hack to run the parallel tests profile only for hadoop-common. - # This code will be removed once hadoop-hdfs is ready for parallel test - # execution. - if [[ ${module} == "hadoop-common-project/hadoop-common" ]] ; then - OPTIONAL_PARALLEL_TESTS_PROFILE=${PARALLEL_TESTS_PROFILE} - else - unset OPTIONAL_PARALLEL_TESTS_PROFILE - fi # shellcheck disable=2086 - echo_and_redirect "${test_logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean install -fae ${NATIVE_PROFILE} ${REQUIRE_TEST_LIB_HADOOP} ${OPTIONAL_PARALLEL_TESTS_PROFILE} ${TESTS_THREAD_COUNT} -D${PROJECT_NAME}PatchProcess + echo_and_redirect "${test_logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean install -fae ${NATIVE_PROFILE} ${REQUIRE_TEST_LIB_HADOOP} ${PARALLEL_TESTS_PROFILE} ${TESTS_THREAD_COUNT} -D${PROJECT_NAME}PatchProcess test_build_result=$? add_jira_footer "${module_suffix} test log" "@@BASE@@/testrun_${module_suffix}.txt" http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 89fe7f4..65eab19 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1541,6 +1541,9 @@ Release 2.8.0 - UNRELEASED DataTransferProtocol sockets and allow configuring auto-tuning (He Tianyi via Colin P. McCabe) + HDFS-9139. Enable parallel JUnit tests for HDFS Pre-commit + (Chris Nauroth and Vinayakumar B via vinayakumarb) + BUG FIXES HDFS-7501. TransactionsSinceLastCheckpoint can be negative on SBNs. http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 016c2bf..f720d0b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -503,14 +503,23 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <artifactId>maven-antrun-plugin</artifactId> <executions> <execution> - <id>create-parallel-tests-dirs</id> + <id>create-parallel-tests-dirs</id> <phase>test-compile</phase> <configuration> <target> - <exec executable="${shell-executable}"> - <arg value="-c"/> - <arg value="for i in {1..${testsThreadCount}}; do mkdir -p ${test.build.data}/$i; mkdir -p ${hadoop.tmp.dir}/$i; done"/> - </exec> + <script language="javascript"><![CDATA[ + var baseDirs = [ + "${test.build.data}", + "${test.build.dir}", + "${hadoop.tmp.dir}" ]; + for (var i in baseDirs) { + for (var j = 1; j <= ${testsThreadCount}; ++j) { + var mkdir = project.createTask("mkdir"); + mkdir.setDir(new java.io.File(baseDirs[i], j)); + mkdir.perform(); + } + } + ]]></script> </target> </configuration> <goals> @@ -524,10 +533,24 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <artifactId>maven-surefire-plugin</artifactId> <configuration> <forkCount>${testsThreadCount}</forkCount> - <argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError -DminiClusterDedicatedDirs=true</argLine> + <reuseForks>false</reuseForks> + <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine> <systemPropertyVariables> <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data> + <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir> <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir> + + <!-- This is intentionally the same directory for all JUnit --> + <!-- forks, for use in the very rare situation that --> + <!-- concurrent tests need to coordinate, such as using lock --> + <!-- files. --> + <test.build.shared.data>${test.build.data}</test.build.shared.data> + + <!-- Due to a Maven quirk, setting this to just --> + <!-- surefire.forkNumber won't do the parameter substitution. --> + <!-- Putting a prefix in front of it like "fork-" makes it --> + <!-- work. --> + <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id> </systemPropertyVariables> </configuration> </plugin> http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSWebHdfsFileContextMainOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSWebHdfsFileContextMainOperations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSWebHdfsFileContextMainOperations.java index 53513fd..02110ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSWebHdfsFileContextMainOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSWebHdfsFileContextMainOperations.java @@ -57,8 +57,6 @@ public class TestSWebHdfsFileContextMainOperations protected static final byte[] data = getFileData(numBlocks, getDefaultBlockSize()); - private static Configuration sslConf; - @BeforeClass public static void clusterSetupAtBeginning() throws IOException, LoginException, URISyntaxException { @@ -67,15 +65,18 @@ public class TestSWebHdfsFileContextMainOperations FileUtil.fullyDelete(base); base.mkdirs(); keystoresDir = new File(BASEDIR).getAbsolutePath(); - sslConf = new Configuration(); - try { sslConfDir = KeyStoreTestUtil .getClasspathDir(TestSWebHdfsFileContextMainOperations.class); - KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, sslConf, false); + KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, CONF, false); + CONF.set(DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + CONF.set(DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); } catch (Exception ex) { throw new RuntimeException(ex); } + CONF.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, "HTTPS_ONLY"); CONF.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); CONF.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java index 9e660b2..37b2a2c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferTestCase.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.protocol.datatransfer.sasl; import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY; @@ -27,6 +28,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTP_POLICY_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY; import static org.junit.Assert.*; @@ -129,6 +131,10 @@ public abstract class SaslDataTransferTestCase { String keystoresDir = baseDir.getAbsolutePath(); String sslConfDir = KeyStoreTestUtil.getClasspathDir(this.getClass()); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); + conf.set(DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + conf.set(DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); return conf; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java index b40e52a..f95594a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java @@ -22,6 +22,7 @@ import static org.junit.Assert.*; import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_PROTECTION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY; @@ -35,6 +36,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KE import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY; import java.io.File; @@ -128,6 +130,10 @@ public class TestSecureNNWithQJM { String sslConfDir = KeyStoreTestUtil.getClasspathDir( TestSecureNNWithQJM.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, baseConf, false); + baseConf.set(DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + baseConf.set(DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); } @AfterClass http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java index b01597a..acd5a47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockScanner.java @@ -612,14 +612,8 @@ public class TestBlockScanner { // We scan 5 bytes per file (1 byte in file, 4 bytes of checksum) final int BYTES_SCANNED_PER_FILE = 5; - final int NUM_FILES[] = new int[] { 1, 5, 10 }; - int TOTAL_FILES = 0; - for (int i = 0; i < NUM_FILES.length; i++) { - TOTAL_FILES += NUM_FILES[i]; - } - ctx.createFiles(0, NUM_FILES[0], 1); - ctx.createFiles(0, NUM_FILES[1], 1); - ctx.createFiles(0, NUM_FILES[2], 1); + int TOTAL_FILES = 16; + ctx.createFiles(0, TOTAL_FILES, 1); // start scanning final TestScanResultHandler.Info info = http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java index 0b47344..5c0a971 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java @@ -277,13 +277,14 @@ public class TestDataNodeMetrics { } for (int x =0; x < 50; x++) { - String s = DFSTestUtil.readFile(fs, new Path("/time.txt." + x)); + DFSTestUtil.readFile(fs, new Path("/time.txt." + x)); } MetricsRecordBuilder rbNew = getMetrics(datanode.getMetrics().name()); long endWriteValue = getLongCounter("TotalWriteTime", rbNew); long endReadValue = getLongCounter("TotalReadTime", rbNew); - + // Lets Metric system update latest metrics + Thread.sleep(100); assertTrue(endReadValue > startReadValue); assertTrue(endWriteValue > startWriteValue); } finally { http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java index af9a6f7..d533cf6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java @@ -263,6 +263,7 @@ public class TestFSNamesystem { } latch.await(); + Thread.sleep(10); // Lets all threads get BLOCKED Assert.assertEquals("Expected number of blocked thread not found", threadCount, rwLock.getQueueLength()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java index 975a664..7dc719e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeHttpServer.java @@ -74,6 +74,10 @@ public class TestNameNodeHttpServer { KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); connectionFactory = URLConnectionFactory .newDefaultURLConnectionFactory(conf); + conf.set(DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + conf.set(DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); } @AfterClass http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java index dca777f..b284ed9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java @@ -221,6 +221,10 @@ public class TestNameNodeRespectsBindHostKeys { LOG.info("Testing behavior without " + DFS_NAMENODE_HTTPS_BIND_HOST_KEY); setupSsl(); + conf.set(DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + conf.set(DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); conf.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java index feec7c8..3b609b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSImage; import org.apache.hadoop.hdfs.server.namenode.NNStorage; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; import org.junit.Test; @@ -127,10 +128,13 @@ public class TestEditLogTailer { // Have to specify IPC ports so the NNs can talk to each other. MiniDFSNNTopology topology = new MiniDFSNNTopology() - .addNameservice(new MiniDFSNNTopology.NSConf("ns1") - .addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(10031)) - .addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(10032)) - .addNN(new MiniDFSNNTopology.NNConf("nn3").setIpcPort(10033))); + .addNameservice(new MiniDFSNNTopology.NSConf("ns1") + .addNN(new MiniDFSNNTopology.NNConf("nn1") + .setIpcPort(ServerSocketUtil.getPort(0, 100))) + .addNN(new MiniDFSNNTopology.NNConf("nn2") + .setIpcPort(ServerSocketUtil.getPort(0, 100))) + .addNN(new MiniDFSNNTopology.NNConf("nn3") + .setIpcPort(ServerSocketUtil.getPort(0, 100)))); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .nnTopology(topology) @@ -152,13 +156,17 @@ public class TestEditLogTailer { long startTxId) throws Exception { URI sharedUri = cluster.getSharedEditsDir(0, 2); File sharedDir = new File(sharedUri.getPath(), "current"); - final File expectedLog = new File(sharedDir, - NNStorage.getInProgressEditsFileName(startTxId)); - + final File expectedInProgressLog = + new File(sharedDir, NNStorage.getInProgressEditsFileName(startTxId)); + final File expectedFinalizedLog = new File(sharedDir, + NNStorage.getFinalizedEditsFileName(startTxId, startTxId + 1)); + // There is a chance that multiple rolling happens by multiple NameNodes + // And expected inprogress file would have also finalized. So look for the + // finalized edits file as well GenericTestUtils.waitFor(new Supplier<Boolean>() { @Override public Boolean get() { - return expectedLog.exists(); + return expectedInProgressLog.exists() || expectedFinalizedLog.exists(); } }, 100, 10000); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java index 7b7100b..1fccb3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java @@ -123,7 +123,7 @@ public class TestHAAppend { if (!isTruncateReady) { TestFileTruncate.checkBlockRecovery(fileToTruncate, - cluster.getFileSystem(1)); + cluster.getFileSystem(1), 300, 200); } AppendTestUtil.checkFullFile(fs, fileToTruncate, truncatePos[0], data, fileToTruncate.toString()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestSeveralNameNodes.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestSeveralNameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestSeveralNameNodes.java index dbe8070..bda928f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestSeveralNameNodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestSeveralNameNodes.java @@ -44,7 +44,7 @@ public class TestSeveralNameNodes { private static final Log LOG = LogFactory.getLog(TestSeveralNameNodes.class); /** ms between failovers between NNs */ - private static final int TIME_BETWEEN_FAILOVERS = 200; + private static final int TIME_BETWEEN_FAILOVERS = 1000; private static final int NUM_NAMENODES = 3; private static final int NUM_THREADS = 3; private static final int LIST_LENGTH = 50; @@ -82,7 +82,7 @@ public class TestSeveralNameNodes { for (int i = 0; i < writers.size(); i++) { CircularWriter writer = writers.get(i); // remove the writer from the ones to check - if (writer.done.await(10, TimeUnit.MILLISECONDS)) { + if (writer.done.await(100, TimeUnit.MILLISECONDS)) { writers.remove(i--); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java index 8cee88f..6d541e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestHttpsFileSystem.java @@ -62,6 +62,10 @@ public class TestHttpsFileSystem { sslConfDir = KeyStoreTestUtil.getClasspathDir(TestHttpsFileSystem.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); + conf.set(DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + conf.set(DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java index 6930835..5e8568c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java @@ -214,7 +214,11 @@ public class TestWebHdfsTokens { keystoresDir = new File(BASEDIR).getAbsolutePath(); sslConfDir = KeyStoreTestUtil.getClasspathDir(TestWebHdfsTokens.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, clusterConf, false); - + clusterConf.set(DFSConfigKeys.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getClientSSLConfigFileName()); + clusterConf.set(DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, + KeyStoreTestUtil.getServerSSLConfigFileName()); + // trick the NN into thinking security is enabled w/o it trying // to login from a keytab UserGroupInformation.setConfiguration(clusterConf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/39581e3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestClientCredentialTimeBasedTokenRefresher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestClientCredentialTimeBasedTokenRefresher.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestClientCredentialTimeBasedTokenRefresher.java index c259b30..d796753 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestClientCredentialTimeBasedTokenRefresher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/oauth2/TestClientCredentialTimeBasedTokenRefresher.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.web.oauth2; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.ServerSocketUtil; import org.apache.hadoop.util.Timer; import org.apache.http.HttpStatus; import org.codehaus.jackson.map.ObjectMapper; @@ -74,7 +75,7 @@ public class TestClientCredentialTimeBasedTokenRefresher { @Test public void refreshUrlIsCorrect() throws IOException { - final int PORT = 7552; + final int PORT = ServerSocketUtil.getPort(0, 20); final String REFRESH_ADDRESS = "http://localhost:"; + PORT + "/refresh"; long tokenExpires = 0;
