Repository: hadoop Updated Branches: refs/heads/trunk f24452d14 -> 2463666ec
HDFS-6184. Capture NN's thread dump when it fails over. Contributed by Ming Ma. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2463666e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2463666e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2463666e Branch: refs/heads/trunk Commit: 2463666ecb553dbde1b8c540a21ad3d599239acf Parents: f24452d Author: Akira Ajisaka <[email protected]> Authored: Wed May 13 11:37:22 2015 +0900 Committer: Akira Ajisaka <[email protected]> Committed: Wed May 13 11:37:22 2015 +0900 ---------------------------------------------------------------------- .../apache/hadoop/ha/ZKFailoverController.java | 5 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +- .../hdfs/tools/DFSZKFailoverController.java | 60 +++++ .../src/main/resources/hdfs-default.xml | 11 + .../ha/TestDFSZKFailoverController.java | 226 ----------------- .../hdfs/tools/TestDFSZKFailoverController.java | 243 +++++++++++++++++++ 7 files changed, 322 insertions(+), 230 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2463666e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java index 9eb1ff8..788d48e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java @@ -844,12 +844,11 @@ public abstract class ZKFailoverController { * @return the last health state passed to the FC * by the HealthMonitor. */ - @VisibleForTesting - synchronized State getLastHealthState() { + protected synchronized State getLastHealthState() { return lastHealthState; } - private synchronized void setLastHealthState(HealthMonitor.State newState) { + protected synchronized void setLastHealthState(HealthMonitor.State newState) { LOG.info("Local service " + localTarget + " entered state: " + newState); lastHealthState = newState; http://git-wip-us.apache.org/repos/asf/hadoop/blob/2463666e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index cd477af..135b50c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -546,6 +546,9 @@ Release 2.8.0 - UNRELEASED HDFS-8255. Rename getBlockReplication to getPreferredBlockReplication. (Contributed by Zhe Zhang) + HDFS-6184. Capture NN's thread dump when it fails over. + (Ming Ma via aajisaka) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/2463666e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 4356b9b..f8e9f3a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -543,7 +543,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final boolean DFS_HA_AUTO_FAILOVER_ENABLED_DEFAULT = false; public static final String DFS_HA_ZKFC_PORT_KEY = "dfs.ha.zkfc.port"; public static final int DFS_HA_ZKFC_PORT_DEFAULT = 8019; - + public static final String DFS_HA_ZKFC_NN_HTTP_TIMEOUT_KEY = "dfs.ha.zkfc.nn.http.timeout.ms"; + public static final int DFS_HA_ZKFC_NN_HTTP_TIMEOUT_KEY_DEFAULT = 20000; + // Security-related configs public static final String DFS_ENCRYPT_DATA_TRANSFER_KEY = "dfs.encrypt.data.transfer"; public static final boolean DFS_ENCRYPT_DATA_TRANSFER_DEFAULT = false; http://git-wip-us.apache.org/repos/asf/hadoop/blob/2463666e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java index 4e256a2..f125a27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSZKFailoverController.java @@ -20,15 +20,20 @@ package org.apache.hadoop.hdfs.tools; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.net.HttpURLConnection; import java.net.InetSocketAddress; +import java.net.URL; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ha.HAServiceTarget; +import org.apache.hadoop.ha.HealthMonitor; import org.apache.hadoop.ha.ZKFailoverController; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; @@ -37,6 +42,7 @@ import org.apache.hadoop.hdfs.HDFSPolicyProvider; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.proto.HAZKInfoProtos.ActiveNodeInfo; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.SecurityUtil; @@ -57,6 +63,9 @@ public class DFSZKFailoverController extends ZKFailoverController { /* the same as superclass's localTarget, but with the more specfic NN type */ private final NNHAServiceTarget localNNTarget; + // This is used only for unit tests + private boolean isThreadDumpCaptured = false; + @Override protected HAServiceTarget dataToTarget(byte[] data) { ActiveNodeInfo proto; @@ -201,4 +210,55 @@ public class DFSZKFailoverController extends ZKFailoverController { LOG.warn(msg); throw new AccessControlException(msg); } + + /** + * capture local NN's thread dump and write it to ZKFC's log. + */ + private void getLocalNNThreadDump() { + isThreadDumpCaptured = false; + // We use the same timeout value for both connection establishment + // timeout and read timeout. + int httpTimeOut = conf.getInt( + DFSConfigKeys.DFS_HA_ZKFC_NN_HTTP_TIMEOUT_KEY, + DFSConfigKeys.DFS_HA_ZKFC_NN_HTTP_TIMEOUT_KEY_DEFAULT); + if (httpTimeOut == 0) { + // If timeout value is set to zero, the feature is turned off. + return; + } + try { + String stacksUrl = DFSUtil.getInfoServer(localNNTarget.getAddress(), + conf, DFSUtil.getHttpClientScheme(conf)) + "/stacks"; + URL url = new URL(stacksUrl); + HttpURLConnection conn = (HttpURLConnection)url.openConnection(); + conn.setReadTimeout(httpTimeOut); + conn.setConnectTimeout(httpTimeOut); + conn.connect(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IOUtils.copyBytes(conn.getInputStream(), out, 4096, true); + StringBuilder localNNThreadDumpContent = + new StringBuilder("-- Local NN thread dump -- \n"); + localNNThreadDumpContent.append(out); + localNNThreadDumpContent.append("\n -- Local NN thread dump -- "); + LOG.info(localNNThreadDumpContent); + isThreadDumpCaptured = true; + } catch (IOException e) { + LOG.warn("Can't get local NN thread dump due to " + e.getMessage()); + } + } + + @Override + protected synchronized void setLastHealthState(HealthMonitor.State newState) { + super.setLastHealthState(newState); + // Capture local NN thread dump when the target NN health state changes. + if (getLastHealthState() == HealthMonitor.State.SERVICE_NOT_RESPONDING || + getLastHealthState() == HealthMonitor.State.SERVICE_UNHEALTHY) { + getLocalNNThreadDump(); + } + } + + @VisibleForTesting + boolean isThreadDumpCaptured() { + return isThreadDumpCaptured; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2463666e/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index fe1d1de..5d1d670 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2334,4 +2334,15 @@ </description> </property> +<property> + <name>dfs.ha.zkfc.nn.http.timeout.ms</name> + <value>20000</value> + <description> + The HTTP connection and read timeout value (unit is ms ) when DFS ZKFC + tries to get local NN thread dump after local NN becomes + SERVICE_NOT_RESPONDING or SERVICE_UNHEALTHY. + If it is set to zero, DFS ZKFC won't get local NN thread dump. + </description> +</property> + </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/2463666e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java deleted file mode 100644 index bcbd543..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java +++ /dev/null @@ -1,226 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.namenode.ha; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.concurrent.TimeoutException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.ha.ClientBaseWithFixes; -import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; -import org.apache.hadoop.ha.HealthMonitor; -import org.apache.hadoop.ha.TestNodeFencer.AlwaysSucceedFencer; -import org.apache.hadoop.ha.ZKFCTestUtil; -import org.apache.hadoop.ha.ZKFailoverController; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.MiniDFSNNTopology; -import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; -import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.hdfs.tools.DFSHAAdmin; -import org.apache.hadoop.hdfs.tools.DFSZKFailoverController; -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.test.MultithreadedTestUtil.TestContext; -import org.apache.hadoop.test.MultithreadedTestUtil.TestingThread; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.google.common.base.Supplier; - -public class TestDFSZKFailoverController extends ClientBaseWithFixes { - private Configuration conf; - private MiniDFSCluster cluster; - private TestContext ctx; - private ZKFCThread thr1, thr2; - private FileSystem fs; - - static { - // Make tests run faster by avoiding fsync() - EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); - } - - @Before - public void setup() throws Exception { - conf = new Configuration(); - // Specify the quorum per-nameservice, to ensure that these configs - // can be nameservice-scoped. - conf.set(ZKFailoverController.ZK_QUORUM_KEY + ".ns1", hostPort); - conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, - AlwaysSucceedFencer.class.getName()); - conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true); - - // Turn off IPC client caching, so that the suite can handle - // the restart of the daemons between test cases. - conf.setInt( - CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, - 0); - - conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn1", 10023); - conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn2", 10024); - - MiniDFSNNTopology topology = new MiniDFSNNTopology() - .addNameservice(new MiniDFSNNTopology.NSConf("ns1") - .addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(10021)) - .addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(10022))); - cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(topology) - .numDataNodes(0) - .build(); - cluster.waitActive(); - - ctx = new TestContext(); - ctx.addThread(thr1 = new ZKFCThread(ctx, 0)); - assertEquals(0, thr1.zkfc.run(new String[]{"-formatZK"})); - - thr1.start(); - waitForHAState(0, HAServiceState.ACTIVE); - - ctx.addThread(thr2 = new ZKFCThread(ctx, 1)); - thr2.start(); - - // Wait for the ZKFCs to fully start up - ZKFCTestUtil.waitForHealthState(thr1.zkfc, - HealthMonitor.State.SERVICE_HEALTHY, ctx); - ZKFCTestUtil.waitForHealthState(thr2.zkfc, - HealthMonitor.State.SERVICE_HEALTHY, ctx); - - fs = HATestUtil.configureFailoverFs(cluster, conf); - } - - @After - public void shutdown() throws Exception { - cluster.shutdown(); - - if (thr1 != null) { - thr1.interrupt(); - } - if (thr2 != null) { - thr2.interrupt(); - } - if (ctx != null) { - ctx.stop(); - } - } - - /** - * Test that automatic failover is triggered by shutting the - * active NN down. - */ - @Test(timeout=60000) - public void testFailoverAndBackOnNNShutdown() throws Exception { - Path p1 = new Path("/dir1"); - Path p2 = new Path("/dir2"); - - // Write some data on the first NN - fs.mkdirs(p1); - // Shut it down, causing automatic failover - cluster.shutdownNameNode(0); - // Data should still exist. Write some on the new NN - assertTrue(fs.exists(p1)); - fs.mkdirs(p2); - assertEquals(AlwaysSucceedFencer.getLastFencedService().getAddress(), - thr1.zkfc.getLocalTarget().getAddress()); - - // Start the first node back up - cluster.restartNameNode(0); - // This should have no effect -- the new node should be STANDBY. - waitForHAState(0, HAServiceState.STANDBY); - assertTrue(fs.exists(p1)); - assertTrue(fs.exists(p2)); - // Shut down the second node, which should failback to the first - cluster.shutdownNameNode(1); - waitForHAState(0, HAServiceState.ACTIVE); - - // First node should see what was written on the second node while it was down. - assertTrue(fs.exists(p1)); - assertTrue(fs.exists(p2)); - assertEquals(AlwaysSucceedFencer.getLastFencedService().getAddress(), - thr2.zkfc.getLocalTarget().getAddress()); - } - - @Test(timeout=30000) - public void testManualFailover() throws Exception { - thr2.zkfc.getLocalTarget().getZKFCProxy(conf, 15000).gracefulFailover(); - waitForHAState(0, HAServiceState.STANDBY); - waitForHAState(1, HAServiceState.ACTIVE); - - thr1.zkfc.getLocalTarget().getZKFCProxy(conf, 15000).gracefulFailover(); - waitForHAState(0, HAServiceState.ACTIVE); - waitForHAState(1, HAServiceState.STANDBY); - } - - @Test(timeout=30000) - public void testManualFailoverWithDFSHAAdmin() throws Exception { - DFSHAAdmin tool = new DFSHAAdmin(); - tool.setConf(conf); - assertEquals(0, - tool.run(new String[]{"-failover", "nn1", "nn2"})); - waitForHAState(0, HAServiceState.STANDBY); - waitForHAState(1, HAServiceState.ACTIVE); - assertEquals(0, - tool.run(new String[]{"-failover", "nn2", "nn1"})); - waitForHAState(0, HAServiceState.ACTIVE); - waitForHAState(1, HAServiceState.STANDBY); - } - - private void waitForHAState(int nnidx, final HAServiceState state) - throws TimeoutException, InterruptedException { - final NameNode nn = cluster.getNameNode(nnidx); - GenericTestUtils.waitFor(new Supplier<Boolean>() { - @Override - public Boolean get() { - try { - return nn.getRpcServer().getServiceStatus().getState() == state; - } catch (Exception e) { - e.printStackTrace(); - return false; - } - } - }, 50, 15000); - } - - /** - * Test-thread which runs a ZK Failover Controller corresponding - * to a given NameNode in the minicluster. - */ - private class ZKFCThread extends TestingThread { - private final DFSZKFailoverController zkfc; - - public ZKFCThread(TestContext ctx, int idx) { - super(ctx); - this.zkfc = DFSZKFailoverController.create( - cluster.getConfiguration(idx)); - } - - @Override - public void doWork() throws Exception { - try { - assertEquals(0, zkfc.run(new String[0])); - } catch (InterruptedException ie) { - // Interrupted by main thread, that's OK. - } - } - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2463666e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java new file mode 100644 index 0000000..3e1c96f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java @@ -0,0 +1,243 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.concurrent.TimeoutException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ha.ClientBaseWithFixes; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; +import org.apache.hadoop.ha.HealthMonitor; +import org.apache.hadoop.ha.TestNodeFencer.AlwaysSucceedFencer; +import org.apache.hadoop.ha.ZKFCTestUtil; +import org.apache.hadoop.ha.ZKFailoverController; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.MiniDFSNNTopology; +import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; +import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.test.MultithreadedTestUtil.TestContext; +import org.apache.hadoop.test.MultithreadedTestUtil.TestingThread; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.base.Supplier; +import org.mockito.Mockito; + +public class TestDFSZKFailoverController extends ClientBaseWithFixes { + private Configuration conf; + private MiniDFSCluster cluster; + private TestContext ctx; + private ZKFCThread thr1, thr2; + private FileSystem fs; + + static { + // Make tests run faster by avoiding fsync() + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } + + @Before + public void setup() throws Exception { + conf = new Configuration(); + // Specify the quorum per-nameservice, to ensure that these configs + // can be nameservice-scoped. + conf.set(ZKFailoverController.ZK_QUORUM_KEY + ".ns1", hostPort); + conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY, + AlwaysSucceedFencer.class.getName()); + conf.setBoolean(DFSConfigKeys.DFS_HA_AUTO_FAILOVER_ENABLED_KEY, true); + + // Turn off IPC client caching, so that the suite can handle + // the restart of the daemons between test cases. + conf.setInt( + CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, + 0); + + conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn1", 10023); + conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn2", 10024); + + MiniDFSNNTopology topology = new MiniDFSNNTopology() + .addNameservice(new MiniDFSNNTopology.NSConf("ns1") + .addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(10021)) + .addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(10022))); + cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(topology) + .numDataNodes(0) + .build(); + cluster.waitActive(); + + ctx = new TestContext(); + ctx.addThread(thr1 = new ZKFCThread(ctx, 0)); + assertEquals(0, thr1.zkfc.run(new String[]{"-formatZK"})); + + thr1.start(); + waitForHAState(0, HAServiceState.ACTIVE); + + ctx.addThread(thr2 = new ZKFCThread(ctx, 1)); + thr2.start(); + + // Wait for the ZKFCs to fully start up + ZKFCTestUtil.waitForHealthState(thr1.zkfc, + HealthMonitor.State.SERVICE_HEALTHY, ctx); + ZKFCTestUtil.waitForHealthState(thr2.zkfc, + HealthMonitor.State.SERVICE_HEALTHY, ctx); + + fs = HATestUtil.configureFailoverFs(cluster, conf); + } + + @After + public void shutdown() throws Exception { + cluster.shutdown(); + + if (thr1 != null) { + thr1.interrupt(); + } + if (thr2 != null) { + thr2.interrupt(); + } + if (ctx != null) { + ctx.stop(); + } + } + + /** + * Test that thread dump is captured after NN state changes. + */ + @Test(timeout=60000) + public void testThreadDumpCaptureAfterNNStateChange() throws Exception { + NameNodeResourceChecker mockResourceChecker = Mockito.mock( + NameNodeResourceChecker.class); + Mockito.doReturn(false).when(mockResourceChecker).hasAvailableDiskSpace(); + cluster.getNameNode(0).getNamesystem() + .setNNResourceChecker(mockResourceChecker); + waitForHAState(0, HAServiceState.STANDBY); + while (!thr1.zkfc.isThreadDumpCaptured()) { + Thread.sleep(1000); + } + } + + /** + * Test that automatic failover is triggered by shutting the + * active NN down. + */ + @Test(timeout=60000) + public void testFailoverAndBackOnNNShutdown() throws Exception { + Path p1 = new Path("/dir1"); + Path p2 = new Path("/dir2"); + + // Write some data on the first NN + fs.mkdirs(p1); + // Shut it down, causing automatic failover + cluster.shutdownNameNode(0); + // Data should still exist. Write some on the new NN + assertTrue(fs.exists(p1)); + fs.mkdirs(p2); + assertEquals(AlwaysSucceedFencer.getLastFencedService().getAddress(), + thr1.zkfc.getLocalTarget().getAddress()); + + // Start the first node back up + cluster.restartNameNode(0); + // This should have no effect -- the new node should be STANDBY. + waitForHAState(0, HAServiceState.STANDBY); + assertTrue(fs.exists(p1)); + assertTrue(fs.exists(p2)); + // Shut down the second node, which should failback to the first + cluster.shutdownNameNode(1); + waitForHAState(0, HAServiceState.ACTIVE); + + // First node should see what was written on the second node while it was down. + assertTrue(fs.exists(p1)); + assertTrue(fs.exists(p2)); + assertEquals(AlwaysSucceedFencer.getLastFencedService().getAddress(), + thr2.zkfc.getLocalTarget().getAddress()); + } + + @Test(timeout=30000) + public void testManualFailover() throws Exception { + thr2.zkfc.getLocalTarget().getZKFCProxy(conf, 15000).gracefulFailover(); + waitForHAState(0, HAServiceState.STANDBY); + waitForHAState(1, HAServiceState.ACTIVE); + + thr1.zkfc.getLocalTarget().getZKFCProxy(conf, 15000).gracefulFailover(); + waitForHAState(0, HAServiceState.ACTIVE); + waitForHAState(1, HAServiceState.STANDBY); + } + + @Test(timeout=30000) + public void testManualFailoverWithDFSHAAdmin() throws Exception { + DFSHAAdmin tool = new DFSHAAdmin(); + tool.setConf(conf); + assertEquals(0, + tool.run(new String[]{"-failover", "nn1", "nn2"})); + waitForHAState(0, HAServiceState.STANDBY); + waitForHAState(1, HAServiceState.ACTIVE); + assertEquals(0, + tool.run(new String[]{"-failover", "nn2", "nn1"})); + waitForHAState(0, HAServiceState.ACTIVE); + waitForHAState(1, HAServiceState.STANDBY); + } + + private void waitForHAState(int nnidx, final HAServiceState state) + throws TimeoutException, InterruptedException { + final NameNode nn = cluster.getNameNode(nnidx); + GenericTestUtils.waitFor(new Supplier<Boolean>() { + @Override + public Boolean get() { + try { + return nn.getRpcServer().getServiceStatus().getState() == state; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + }, 50, 15000); + } + + /** + * Test-thread which runs a ZK Failover Controller corresponding + * to a given NameNode in the minicluster. + */ + private class ZKFCThread extends TestingThread { + private final DFSZKFailoverController zkfc; + + public ZKFCThread(TestContext ctx, int idx) { + super(ctx); + this.zkfc = DFSZKFailoverController.create( + cluster.getConfiguration(idx)); + } + + @Override + public void doWork() throws Exception { + try { + assertEquals(0, zkfc.run(new String[0])); + } catch (InterruptedException ie) { + // Interrupted by main thread, that's OK. + } + } + } + +}
