Author: bikas
Date: Fri Oct 4 20:37:47 2013
New Revision: 1529285
URL: http://svn.apache.org/r1529285
Log:
Merge r1529251 from trunk to branch-2 for YARN-1232. Configuration to support
multiple RMs (Karthik Kambatla via bikas)
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
- copied unchanged from r1529251,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
- copied unchanged from r1529251,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAProtocolService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1529285&r1=1529284&r2=1529285&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Fri Oct 4
20:37:47 2013
@@ -33,6 +33,9 @@ Release 2.3.0 - UNRELEASED
YARN-1199. Make NM/RM Versions Available (Mit Desai via jeagles)
+ YARN-1232. Configuration to support multiple RMs (Karthik Kambatla via
+ bikas)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1529285&r1=1529284&r2=1529285&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
Fri Oct 4 20:37:47 2013
@@ -18,15 +18,12 @@
package org.apache.hadoop.yarn.conf;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -84,7 +81,7 @@ public class YarnConfiguration extends C
// Resource Manager Configs
////////////////////////////////
public static final String RM_PREFIX = "yarn.resourcemanager.";
-
+
/** The address of the applications manager interface in the RM.*/
public static final String RM_ADDRESS =
RM_PREFIX + "address";
@@ -281,6 +278,8 @@ public class YarnConfiguration extends C
public static final String RM_HA_ENABLED = RM_HA_PREFIX + "enabled";
public static final boolean DEFAULT_RM_HA_ENABLED = false;
+ public static final String RM_HA_IDS = RM_HA_PREFIX + "rm-ids";
+ public static final String RM_HA_ID = RM_HA_PREFIX + "id";
////////////////////////////////
// RM state store configs
@@ -854,4 +853,24 @@ public class YarnConfiguration extends C
this.reloadConfiguration();
}
}
+
+ /**
+ * Get the socket address for <code>name</code> property as a
+ * <code>InetSocketAddress</code>.
+ * @param name property name.
+ * @param defaultAddress the default value
+ * @param defaultPort the default port
+ * @return InetSocketAddress
+ */
+ @Override
+ public InetSocketAddress getSocketAddr(
+ String name, String defaultAddress, int defaultPort) {
+ String address;
+ if (HAUtil.isHAEnabled(this)) {
+ address = HAUtil.getConfValueForRMInstance(name, defaultAddress, this);
+ } else {
+ address = get(name, defaultAddress);
+ }
+ return NetUtils.createSocketAddr(address, defaultPort, name);
+ }
}
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java?rev=1529285&r1=1529284&r2=1529285&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
Fri Oct 4 20:37:47 2013
@@ -38,8 +38,11 @@ public class ClientRMProxy<T> extends RM
private static final Log LOG = LogFactory.getLog(ClientRMProxy.class);
- public static <T> T createRMProxy(final Configuration conf,
+ public static <T> T createRMProxy(final Configuration configuration,
final Class<T> protocol) throws IOException {
+ YarnConfiguration conf = (configuration instanceof YarnConfiguration)
+ ? (YarnConfiguration) configuration
+ : new YarnConfiguration(configuration);
InetSocketAddress rmAddress = getRMAddress(conf, protocol);
return createRMProxy(conf, protocol, rmAddress);
}
@@ -60,7 +63,7 @@ public class ClientRMProxy<T> extends RM
}
}
- private static InetSocketAddress getRMAddress(Configuration conf,
+ private static InetSocketAddress getRMAddress(YarnConfiguration conf,
Class<?> protocol) throws IOException {
if (protocol == ApplicationClientProtocol.class) {
return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1529285&r1=1529284&r2=1529285&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
Fri Oct 4 20:37:47 2013
@@ -320,14 +320,35 @@
</property>
<property>
- <description>Enable RM high-availability. When enabled, the RM starts
- in the Standby mode by default, and transitions to the Active mode when
- prompted to.</description>
+ <description>Enable RM high-availability. When enabled,
+ (1) The RM starts in the Standby mode by default, and transitions to
+ the Active mode when prompted to.
+ (2) The nodes in the RM ensemble are listed in
+ yarn.resourcemanager.ha.rm-ids
+ (3) The id of each RM comes from yarn.resourcemanager.ha.id
+ (4) The actual physical addresses come from the configs of the pattern
+ - {rpc-config}.{id}</description>
<name>yarn.resourcemanager.ha.enabled</name>
<value>false</value>
</property>
<property>
+ <description>The list of RM nodes in the cluster when HA is
+ enabled. See description of yarn.resourcemanager.ha
+ .enabled for full details on how this is used.</description>
+ <name>yarn.resourcemanager.ha.rm-ids</name>
+ <!--value>rm1,rm2</value-->
+ </property>
+
+ <property>
+ <description>The id (string) of the current RM. When HA is enabled, this
+ is a required config. See description of yarn.resourcemanager.ha.enabled
+ for full details on how this is used.</description>
+ <name>yarn.resourcemanager.ha.id</name>
+ <!--value>rm1</value-->
+ </property>
+
+ <property>
<description>The maximum number of completed applications RM keeps.
</description>
<name>yarn.resourcemanager.max-completed-applications</name>
<value>10000</value>
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java?rev=1529285&r1=1529284&r2=1529285&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
Fri Oct 4 20:37:47 2013
@@ -31,13 +31,17 @@ public class ServerRMProxy<T> extends RM
private static final Log LOG = LogFactory.getLog(ServerRMProxy.class);
- public static <T> T createRMProxy(final Configuration conf,
+ public static <T> T createRMProxy(final Configuration configuration,
final Class<T> protocol) throws IOException {
+ YarnConfiguration conf = (configuration instanceof YarnConfiguration)
+ ? (YarnConfiguration) configuration
+ : new YarnConfiguration(configuration);
InetSocketAddress rmAddress = getRMAddress(conf, protocol);
return createRMProxy(conf, protocol, rmAddress);
}
- private static InetSocketAddress getRMAddress(Configuration conf, Class<?>
protocol) {
+ private static InetSocketAddress getRMAddress(YarnConfiguration conf,
+ Class<?> protocol) {
if (protocol == ResourceTracker.class) {
return conf.getSocketAddr(
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAProtocolService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAProtocolService.java?rev=1529285&r1=1529284&r2=1529285&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAProtocolService.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAProtocolService.java
Fri Oct 4 20:37:47 2013
@@ -29,8 +29,8 @@ import org.apache.hadoop.ha.HAServicePro
import org.apache.hadoop.ha.HAServiceStatus;
import org.apache.hadoop.ha.HealthCheckFailedException;
import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.conf.HAUtil;
import java.io.IOException;
@@ -44,6 +44,7 @@ public class RMHAProtocolService extends
private ResourceManager rm;
@VisibleForTesting
protected HAServiceState haState = HAServiceState.INITIALIZING;
+ private boolean haEnabled;
public RMHAProtocolService(ResourceManager resourceManager) {
super("RMHAProtocolService");
@@ -51,17 +52,20 @@ public class RMHAProtocolService extends
}
@Override
- public synchronized void serviceInit(Configuration conf) throws Exception {
+ protected synchronized void serviceInit(Configuration conf) throws
+ Exception {
this.conf = conf;
+ haEnabled = HAUtil.isHAEnabled(this.conf);
+ if (haEnabled) {
+ HAUtil.setAllRpcAddresses(this.conf);
+ rm.setConf(this.conf);
+ }
rm.createAndInitActiveServices();
super.serviceInit(this.conf);
}
@Override
- public synchronized void serviceStart() throws Exception {
- boolean haEnabled = this.conf.getBoolean(YarnConfiguration.RM_HA_ENABLED,
- YarnConfiguration.DEFAULT_RM_HA_ENABLED);
-
+ protected synchronized void serviceStart() throws Exception {
if (haEnabled) {
transitionToStandby(true);
} else {
@@ -72,7 +76,7 @@ public class RMHAProtocolService extends
}
@Override
- public synchronized void serviceStop() throws Exception {
+ protected synchronized void serviceStop() throws Exception {
transitionToStandby(false);
haState = HAServiceState.STOPPING;
super.serviceStop();
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1529285&r1=1529284&r2=1529285&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
Fri Oct 4 20:37:47 2013
@@ -731,6 +731,10 @@ public class ResourceManager extends Com
webApp = builder.start(new RMWebApp(this));
}
+ void setConf(Configuration configuration) {
+ conf = configuration;
+ }
+
/**
* Helper method to create and init {@link #activeServices}. This creates an
* instance of {@link RMActiveServices} and initializes it.
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java?rev=1529285&r1=1529284&r2=1529285&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
Fri Oct 4 20:37:47 2013
@@ -26,6 +26,7 @@ import org.apache.hadoop.ha.HAServicePro
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.ha.HealthCheckFailedException;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.conf.HAUtil;
import org.junit.Before;
import org.junit.Test;
@@ -42,10 +43,19 @@ public class TestRMHA {
private static final String STATE_ERR =
"ResourceManager is in wrong HA state";
+ private static final String RM1_ADDRESS = "0.0.0.0:0";
+ private static final String RM1_NODE_ID = "rm1";
+
@Before
public void setUp() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID);
+ for (String confKey : HAUtil.RPC_ADDRESS_CONF_KEYS) {
+ conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
+ }
+ conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
+
rm = new MockRM(conf);
rm.init(conf);
}
