Author: todd
Date: Fri Mar 23 06:29:40 2012
New Revision: 1304203
URL: http://svn.apache.org/viewvc?rev=1304203&view=rev
Log:
HDFS-3071. haadmin failover command does not provide enough detail when target
NN is not ready to be active. Contributed by Todd Lipcon.
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1304203&r1=1304202&r2=1304203&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Mar 23
06:29:40 2012
@@ -255,6 +255,9 @@ Release 0.23.3 - UNRELEASED
HDFS-3044. fsck move should be non-destructive by default.
(Colin Patrick McCabe via eli)
+ HDFS-3071. haadmin failover command does not provide enough detail when
+ target NN is not ready to be active. (todd)
+
OPTIMIZATIONS
HDFS-3024. Improve performance of stringification in addStoredBlock (todd)
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1304203&r1=1304202&r2=1304203&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Fri Mar 23 06:29:40 2012
@@ -32,6 +32,7 @@ import org.apache.hadoop.HadoopIllegalAr
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.ha.HAServiceStatus;
import org.apache.hadoop.ha.HealthCheckFailedException;
import org.apache.hadoop.ha.ServiceFailedException;
import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -990,24 +991,41 @@ public class NameNode {
state.setState(haContext, STANDBY_STATE);
}
- synchronized HAServiceState getServiceState() throws AccessControlException {
+ synchronized HAServiceStatus getServiceStatus()
+ throws ServiceFailedException, AccessControlException {
namesystem.checkSuperuserPrivilege();
+ if (!haEnabled) {
+ throw new ServiceFailedException("HA for namenode is not enabled");
+ }
if (state == null) {
- return HAServiceState.INITIALIZING;
+ return new HAServiceStatus(HAServiceState.INITIALIZING);
}
- return state.getServiceState();
+ HAServiceState retState = state.getServiceState();
+ HAServiceStatus ret = new HAServiceStatus(retState);
+ if (retState == HAServiceState.STANDBY) {
+ String safemodeTip = namesystem.getSafeModeTip();
+ if (!safemodeTip.isEmpty()) {
+ ret.setNotReadyToBecomeActive(
+ "The NameNode is in safemode. " +
+ safemodeTip);
+ } else {
+ ret.setReadyToBecomeActive();
+ }
+ } else if (retState == HAServiceState.ACTIVE) {
+ ret.setReadyToBecomeActive();
+ } else {
+ ret.setNotReadyToBecomeActive("State is " + state);
+ }
+ return ret;
}
- synchronized boolean readyToBecomeActive()
- throws ServiceFailedException, AccessControlException {
- namesystem.checkSuperuserPrivilege();
- if (!haEnabled) {
- throw new ServiceFailedException("HA for namenode is not enabled");
+ synchronized HAServiceState getServiceState() {
+ if (state == null) {
+ return HAServiceState.INITIALIZING;
}
- return !isInSafeMode();
+ return state.getServiceState();
}
-
/**
* Class used as expose {@link NameNode} as context to {@link HAState}
*
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java?rev=1304203&r1=1304202&r2=1304203&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
Fri Mar 23 06:29:40 2012
@@ -41,6 +41,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.ha.HAServiceStatus;
import org.apache.hadoop.ha.HealthCheckFailedException;
import org.apache.hadoop.ha.ServiceFailedException;
import
org.apache.hadoop.ha.proto.HAServiceProtocolProtos.HAServiceProtocolService;
@@ -979,15 +980,9 @@ class NameNodeRpcServer implements Namen
}
@Override // HAServiceProtocol
- public synchronized HAServiceState getServiceState()
- throws AccessControlException {
- return nn.getServiceState();
- }
-
- @Override // HAServiceProtocol
- public synchronized boolean readyToBecomeActive()
- throws ServiceFailedException, AccessControlException {
- return nn.readyToBecomeActive();
+ public synchronized HAServiceStatus getServiceStatus()
+ throws AccessControlException, ServiceFailedException {
+ return nn.getServiceStatus();
}
/**
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java?rev=1304203&r1=1304202&r2=1304203&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
Fri Mar 23 06:29:40 2012
@@ -31,13 +31,13 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.ha.HAServiceStatus;
import org.apache.hadoop.ha.HealthCheckFailedException;
import org.apache.hadoop.ha.NodeFencer;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
-import static org.mockito.Mockito.when;
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
@@ -51,6 +51,11 @@ public class TestDFSHAAdmin {
private HAServiceProtocol mockProtocol;
private static final String NSID = "ns1";
+
+ private static final HAServiceStatus STANDBY_READY_RESULT =
+ new HAServiceStatus(HAServiceState.STANDBY)
+ .setReadyToBecomeActive();
+
private static String HOST_A = "1.2.3.1";
private static String HOST_B = "1.2.3.2";
@@ -73,7 +78,6 @@ public class TestDFSHAAdmin {
@Before
public void setup() throws IOException {
mockProtocol = Mockito.mock(HAServiceProtocol.class);
- when(mockProtocol.readyToBecomeActive()).thenReturn(true);
tool = new DFSHAAdmin() {
@Override
protected HAServiceProtocol getProtocol(String serviceId) throws
IOException {
@@ -105,8 +109,9 @@ public class TestDFSHAAdmin {
@Test
public void testNamenodeResolution() throws Exception {
+
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
assertEquals(0, runTool("-getServiceState", "nn1"));
- Mockito.verify(mockProtocol).getServiceState();
+ Mockito.verify(mockProtocol).getServiceStatus();
assertEquals(-1, runTool("-getServiceState", "undefined"));
assertOutputContains(
"Unable to determine service address for namenode 'undefined'");
@@ -133,13 +138,13 @@ public class TestDFSHAAdmin {
@Test
public void testFailoverWithNoFencerConfigured() throws Exception {
-
Mockito.doReturn(HAServiceState.STANDBY).when(mockProtocol).getServiceState();
+
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
assertEquals(-1, runTool("-failover", "nn1", "nn2"));
}
@Test
public void testFailoverWithFencerConfigured() throws Exception {
-
Mockito.doReturn(HAServiceState.STANDBY).when(mockProtocol).getServiceState();
+
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
tool.setConf(conf);
@@ -148,7 +153,7 @@ public class TestDFSHAAdmin {
@Test
public void testFailoverWithFencerAndNameservice() throws Exception {
-
Mockito.doReturn(HAServiceState.STANDBY).when(mockProtocol).getServiceState();
+
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
tool.setConf(conf);
@@ -157,7 +162,7 @@ public class TestDFSHAAdmin {
@Test
public void testFailoverWithFencerConfiguredAndForce() throws Exception {
-
Mockito.doReturn(HAServiceState.STANDBY).when(mockProtocol).getServiceState();
+
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
tool.setConf(conf);
@@ -166,7 +171,7 @@ public class TestDFSHAAdmin {
@Test
public void testFailoverWithForceActive() throws Exception {
-
Mockito.doReturn(HAServiceState.STANDBY).when(mockProtocol).getServiceState();
+
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
tool.setConf(conf);
@@ -175,7 +180,7 @@ public class TestDFSHAAdmin {
@Test
public void testFailoverWithInvalidFenceArg() throws Exception {
-
Mockito.doReturn(HAServiceState.STANDBY).when(mockProtocol).getServiceState();
+
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
tool.setConf(conf);
@@ -184,13 +189,13 @@ public class TestDFSHAAdmin {
@Test
public void testFailoverWithFenceButNoFencer() throws Exception {
-
Mockito.doReturn(HAServiceState.STANDBY).when(mockProtocol).getServiceState();
+
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
assertEquals(-1, runTool("-failover", "nn1", "nn2", "--forcefence"));
}
@Test
public void testFailoverWithFenceAndBadFencer() throws Exception {
-
Mockito.doReturn(HAServiceState.STANDBY).when(mockProtocol).getServiceState();
+
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
conf.set(NodeFencer.CONF_METHODS_KEY, "foobar!");
tool.setConf(conf);
@@ -199,7 +204,7 @@ public class TestDFSHAAdmin {
@Test
public void testForceFenceOptionListedBeforeArgs() throws Exception {
-
Mockito.doReturn(HAServiceState.STANDBY).when(mockProtocol).getServiceState();
+
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
HdfsConfiguration conf = getHAConf();
conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
tool.setConf(conf);
@@ -207,9 +212,10 @@ public class TestDFSHAAdmin {
}
@Test
- public void testGetServiceState() throws Exception {
+ public void testGetServiceStatus() throws Exception {
+
Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
assertEquals(0, runTool("-getServiceState", "nn1"));
- Mockito.verify(mockProtocol).getServiceState();
+ Mockito.verify(mockProtocol).getServiceStatus();
}
@Test
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java?rev=1304203&r1=1304202&r2=1304203&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
Fri Mar 23 06:29:40 2012
@@ -21,6 +21,7 @@ import static org.junit.Assert.*;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.PrintStream;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.Log;
@@ -28,6 +29,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.ha.NodeFencer;
import org.junit.After;
@@ -46,7 +48,10 @@ public class TestDFSHAAdminMiniCluster {
private MiniDFSCluster cluster;
private Configuration conf;
private DFSHAAdmin tool;
-
+ private ByteArrayOutputStream errOutBytes = new ByteArrayOutputStream();
+
+ private String errOutput;
+
@Before
public void setup() throws IOException {
conf = new Configuration();
@@ -55,6 +60,7 @@ public class TestDFSHAAdminMiniCluster {
.build();
tool = new DFSHAAdmin();
tool.setConf(conf);
+ tool.setErrOut(new PrintStream(errOutBytes));
cluster.waitActive();
}
@@ -67,6 +73,12 @@ public class TestDFSHAAdminMiniCluster {
public void testGetServiceState() throws Exception {
assertEquals(0, runTool("-getServiceState", "nn1"));
assertEquals(0, runTool("-getServiceState", "nn2"));
+
+ cluster.transitionToActive(0);
+ assertEquals(0, runTool("-getServiceState", "nn1"));
+
+ NameNodeAdapter.enterSafeMode(cluster.getNameNode(0), false);
+ assertEquals(0, runTool("-getServiceState", "nn1"));
}
@Test
@@ -85,6 +97,18 @@ public class TestDFSHAAdminMiniCluster {
assertEquals(0, runTool("-transitionToStandby", "nn2"));
assertTrue(nnode2.isStandbyState());
}
+
+ @Test
+ public void testTryFailoverToSafeMode() throws Exception {
+ conf.set(NodeFencer.CONF_METHODS_KEY, "shell(true)");
+ tool.setConf(conf);
+
+ NameNodeAdapter.enterSafeMode(cluster.getNameNode(0), false);
+ assertEquals(-1, runTool("-failover", "nn2", "nn1"));
+ assertTrue("Bad output: " + errOutput,
+ errOutput.contains("is not ready to become active: " +
+ "The NameNode is in safemode"));
+ }
/**
* Test failover with various options
@@ -132,11 +156,10 @@ public class TestDFSHAAdminMiniCluster {
}
private int runTool(String ... args) throws Exception {
- ByteArrayOutputStream errOutBytes = new ByteArrayOutputStream();
errOutBytes.reset();
LOG.info("Running: DFSHAAdmin " + Joiner.on(" ").join(args));
int ret = tool.run(args);
- String errOutput = new String(errOutBytes.toByteArray(), Charsets.UTF_8);
+ errOutput = new String(errOutBytes.toByteArray(), Charsets.UTF_8);
LOG.info("Output:\n" + errOutput);
return ret;
}
