Repository: hadoop Updated Branches: refs/heads/trunk 3c4a34e90 -> 53e3bf7e7
HDFS-9429. Tests in TestDFSAdminWithHA intermittently fail with EOFException (Xiao Chen via Colin P. McCabe) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53e3bf7e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53e3bf7e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53e3bf7e Branch: refs/heads/trunk Commit: 53e3bf7e704c332fb119f55cb92520a51b644bfc Parents: 3c4a34e Author: Colin Patrick Mccabe <[email protected]> Authored: Tue Dec 1 23:21:21 2015 -0800 Committer: Colin Patrick Mccabe <[email protected]> Committed: Tue Dec 1 23:21:21 2015 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../apache/hadoop/hdfs/TestRollingUpgrade.java | 1 + .../hadoop/hdfs/TestRollingUpgradeRollback.java | 1 + .../hdfs/qjournal/MiniJournalCluster.java | 38 ++++++++++++++++++++ .../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 1 + .../hdfs/qjournal/TestMiniJournalCluster.java | 1 + .../hadoop/hdfs/qjournal/TestNNWithQJM.java | 1 + .../hdfs/qjournal/TestSecureNNWithQJM.java | 1 + .../qjournal/client/TestEpochsAreUnique.java | 1 + .../hdfs/qjournal/client/TestQJMWithFaults.java | 3 ++ .../client/TestQuorumJournalManager.java | 1 + .../qjournal/server/TestJournalNodeMXBean.java | 10 +++--- 12 files changed, 58 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e3bf7e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 89aaed4..bb3f148 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2430,6 +2430,9 @@ Release 2.8.0 - UNRELEASED HDFS-6533. TestBPOfferService#testBasicFunctionalitytest fails intermittently. (Wei-Chiu Chuang via Arpit Agarwal) + HDFS-9429. Tests in TestDFSAdminWithHA intermittently fail with + EOFException (Xiao Chen via Colin P. McCabe) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e3bf7e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java index 72e16e4..b3279ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java @@ -168,6 +168,7 @@ public class TestRollingUpgrade { final Configuration conf = new HdfsConfiguration(); final MiniJournalCluster mjc = new MiniJournalCluster.Builder(conf).build(); + mjc.waitActive(); setConf(conf, nn1Dir, mjc); { http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e3bf7e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeRollback.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeRollback.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeRollback.java index 38cfb92..b5ef5ee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeRollback.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgradeRollback.java @@ -153,6 +153,7 @@ public class TestRollingUpgradeRollback { try { mjc = new MiniJournalCluster.Builder(conf).numJournalNodes( NUM_JOURNAL_NODES).build(); + mjc.waitActive(); conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, mjc .getQuorumJournalURI(JOURNAL_ID).toString()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e3bf7e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java index 202188d..7b974c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniJournalCluster.java @@ -17,26 +17,34 @@ */ package org.apache.hadoop.hdfs.qjournal; +import static org.apache.hadoop.hdfs.qjournal.QJMTestUtil.FAKE_NSINFO; +import static org.junit.Assert.fail; + import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.util.List; +import java.util.concurrent.TimeoutException; +import com.google.common.base.Supplier; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager; import org.apache.hadoop.hdfs.qjournal.server.JournalNode; import org.apache.hadoop.net.NetUtils; import com.google.common.base.Joiner; import com.google.common.collect.Lists; +import org.apache.hadoop.test.GenericTestUtils; public class MiniJournalCluster { + public static final String CLUSTER_WAITACTIVE_URI = "waitactive"; public static class Builder { private String baseDir; private int numJournalNodes = 3; @@ -217,4 +225,34 @@ public class MiniJournalCluster { return nodes.length; } + /** + * Wait until all the journalnodes start. + */ + public void waitActive() throws IOException { + for (int i = 0; i < nodes.length; i++) { + final int index = i; + try { + GenericTestUtils.waitFor(new Supplier<Boolean>() { + // wait until all JN's IPC server is running + @Override public Boolean get() { + try { + QuorumJournalManager qjm = + new QuorumJournalManager(nodes[index].node.getConf(), + getQuorumJournalURI(CLUSTER_WAITACTIVE_URI), FAKE_NSINFO); + qjm.hasSomeData(); + qjm.close(); + } catch (IOException e) { + // Exception from IPC call, likely due to server not ready yet. + return false; + } + return true; + } + }, 50, 3000); + } catch (TimeoutException e) { + fail("Time out while waiting for journal node " + index + " to start."); + } catch (InterruptedException ite) { + LOG.warn("Thread interrupted when waiting for node start", ite); + } + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e3bf7e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java index 470a08b..8e838c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java @@ -98,6 +98,7 @@ public class MiniQJMHACluster { // start 3 journal nodes journalCluster = new MiniJournalCluster.Builder(conf).format(true) .build(); + journalCluster.waitActive(); URI journalURI = journalCluster.getQuorumJournalURI(NAMESERVICE); // start cluster with specified NameNodes http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e3bf7e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java index fbb51e1..cace7c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestMiniJournalCluster.java @@ -36,6 +36,7 @@ public class TestMiniJournalCluster { Configuration conf = new Configuration(); MiniJournalCluster c = new MiniJournalCluster.Builder(conf) .build(); + c.waitActive(); try { URI uri = c.getQuorumJournalURI("myjournal"); String[] addrs = uri.getAuthority().split(";"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e3bf7e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java index 7e81b67..d713bc7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java @@ -52,6 +52,7 @@ public class TestNNWithQJM { @Before public void startJNs() throws Exception { mjc = new MiniJournalCluster.Builder(conf).build(); + mjc.waitActive(); } @After http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e3bf7e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java index f95594a..166f18e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java @@ -212,6 +212,7 @@ public class TestSecureNNWithQJM { private void startCluster() throws IOException { mjc = new MiniJournalCluster.Builder(conf) .build(); + mjc.waitActive(); conf.set(DFS_NAMENODE_EDITS_DIR_KEY, mjc.getQuorumJournalURI("myjournal").toString()); cluster = new MiniDFSCluster.Builder(conf) http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e3bf7e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestEpochsAreUnique.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestEpochsAreUnique.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestEpochsAreUnique.java index bd9cf6f..d57e089 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestEpochsAreUnique.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestEpochsAreUnique.java @@ -51,6 +51,7 @@ public class TestEpochsAreUnique { public void testSingleThreaded() throws IOException { Configuration conf = new Configuration(); MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf).build(); + cluster.waitActive(); URI uri = cluster.getQuorumJournalURI(JID); QuorumJournalManager qjm = new QuorumJournalManager( conf, uri, FAKE_NSINFO); http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e3bf7e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java index aac2f49..b0a9b99 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java @@ -98,6 +98,7 @@ public class TestQJMWithFaults { private static long determineMaxIpcNumber() throws Exception { Configuration conf = new Configuration(); MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf).build(); + cluster.waitActive(); QuorumJournalManager qjm = null; long ret; try { @@ -146,6 +147,7 @@ public class TestQJMWithFaults { MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf) .build(); + cluster.waitActive(); QuorumJournalManager qjm = null; try { qjm = createInjectableQJM(cluster); @@ -218,6 +220,7 @@ public class TestQJMWithFaults { MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf) .build(); + cluster.waitActive(); // Format the cluster using a non-faulty QJM. QuorumJournalManager qjmForInitialFormat = http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e3bf7e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java index ad67deb..b9a0924 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java @@ -94,6 +94,7 @@ public class TestQuorumJournalManager { cluster = new MiniJournalCluster.Builder(conf) .build(); + cluster.waitActive(); qjm = createSpyingQJM(); spies = qjm.getLoggerSetForTests().getLoggersForTests(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e3bf7e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java index 3471848..498ef71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournalNodeMXBean.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.qjournal.server; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.lang.management.ManagementFactory; @@ -52,6 +53,7 @@ public class TestJournalNodeMXBean { // start 1 journal node jCluster = new MiniJournalCluster.Builder(new Configuration()).format(true) .numJournalNodes(NUM_JN).build(); + jCluster.waitActive(); jn = jCluster.getJournalNode(0); } @@ -89,19 +91,19 @@ public class TestJournalNodeMXBean { Map<String, String> infoMap = new HashMap<String, String>(); infoMap.put("Formatted", "true"); jMap.put(NAMESERVICE, infoMap); + Map<String, String> infoMap1 = new HashMap<>(); + infoMap1.put("Formatted", "false"); + jMap.put(MiniJournalCluster.CLUSTER_WAITACTIVE_URI, infoMap1); assertEquals(JSON.toString(jMap), journalStatus); // restart journal node without formatting jCluster = new MiniJournalCluster.Builder(new Configuration()).format(false) .numJournalNodes(NUM_JN).build(); + jCluster.waitActive(); jn = jCluster.getJournalNode(0); // re-check journalStatus = (String) mbs.getAttribute(mxbeanName, "JournalsStatus"); assertEquals(jn.getJournalsStatus(), journalStatus); - jMap = new HashMap<String, Map<String, String>>(); - infoMap = new HashMap<String, String>(); - infoMap.put("Formatted", "true"); - jMap.put(NAMESERVICE, infoMap); assertEquals(JSON.toString(jMap), journalStatus); } }
