YARN-6866. Minor clean-up and fixes in anticipation of YARN-2915 merge with trunk. (Botong Huang via Subru).
(cherry picked from commit 40453879ec860819b080ccc82454480a436c0adc) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86f05c33 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86f05c33 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86f05c33 Branch: refs/heads/trunk Commit: 86f05c33125af3b435d6793551680ffa35047aa7 Parents: 8737c2a Author: Subru Krishnan <[email protected]> Authored: Tue Jul 25 20:22:45 2017 -0700 Committer: Carlo Curino <[email protected]> Committed: Tue Aug 1 17:28:28 2017 -0700 ---------------------------------------------------------------------- hadoop-project/pom.xml | 6 +++--- .../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++-- .../hadoop/yarn/conf/TestYarnConfigurationFields.java | 4 ++++ .../router/clientrm/FederationClientInterceptor.java | 13 +++---------- .../hadoop-yarn-site/src/site/markdown/Federation.md | 8 ++++---- 5 files changed, 16 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/86f05c33/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 7301e90..61d7c9b 100755 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -99,9 +99,9 @@ <ldap-api.version>1.0.0-M33</ldap-api.version> <jcache.version>1.0.0</jcache.version> - <ehcache.version>3.0.3</ehcache.version> - <hikari.version>2.4.11</hikari.version> - <mssql.version>6.1.0.jre7</mssql.version> + <ehcache.version>3.3.1</ehcache.version> + <hikari.version>2.4.12</hikari.version> + <mssql.version>6.2.1.jre7</mssql.version> <!-- define the Java language version used by the compiler --> <javac.version>1.8</javac.version> http://git-wip-us.apache.org/repos/asf/hadoop/blob/86f05c33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 0c0dd11..d2013bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2645,7 +2645,7 @@ public class YarnConfiguration extends Configuration { ROUTER_PREFIX + "clientrm."; public static final String ROUTER_CLIENTRM_ADDRESS = - ROUTER_CLIENTRM_PREFIX + ".address"; + ROUTER_CLIENTRM_PREFIX + "address"; public static final int DEFAULT_ROUTER_CLIENTRM_PORT = 8050; public static final String DEFAULT_ROUTER_CLIENTRM_ADDRESS = "0.0.0.0:" + DEFAULT_ROUTER_CLIENTRM_PORT; @@ -2663,7 +2663,7 @@ public class YarnConfiguration extends Configuration { public static final String ROUTER_RMADMIN_PREFIX = ROUTER_PREFIX + "rmadmin."; public static final String ROUTER_RMADMIN_ADDRESS = - ROUTER_RMADMIN_PREFIX + ".address"; + ROUTER_RMADMIN_PREFIX + "address"; public static final int DEFAULT_ROUTER_RMADMIN_PORT = 8052; public static final String DEFAULT_ROUTER_RMADMIN_ADDRESS = "0.0.0.0:" + DEFAULT_ROUTER_RMADMIN_PORT; http://git-wip-us.apache.org/repos/asf/hadoop/blob/86f05c33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index a439cd5..a035dc7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -77,6 +77,10 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase { .add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS); configurationPropsToSkipCompare .add(YarnConfiguration.RM_EPOCH); + configurationPropsToSkipCompare + .add(YarnConfiguration.ROUTER_CLIENTRM_ADDRESS); + configurationPropsToSkipCompare + .add(YarnConfiguration.ROUTER_RMADMIN_ADDRESS); // Federation policies configs to be ignored configurationPropsToSkipCompare http://git-wip-us.apache.org/repos/asf/hadoop/blob/86f05c33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index ecf53ac..7268ebd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.router.clientrm; import java.io.IOException; -import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -88,10 +87,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRespo import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil; import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils; import org.apache.hadoop.yarn.server.federation.policies.RouterPolicyFacade; import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; @@ -175,14 +174,8 @@ public class FederationClientInterceptor ApplicationClientProtocol clientRMProxy = null; try { - clientRMProxy = - user.doAs(new PrivilegedExceptionAction<ApplicationClientProtocol>() { - @Override - public ApplicationClientProtocol run() throws Exception { - return ClientRMProxy.createRMProxy(getConf(), - ApplicationClientProtocol.class); - } - }); + clientRMProxy = FederationProxyProviderUtil.createRMProxy(getConf(), + ApplicationClientProtocol.class, subClusterId, user); } catch (Exception e) { RouterServerUtil.logAndThrowException( "Unable to create the interface to reach the SubCluster " http://git-wip-us.apache.org/repos/asf/hadoop/blob/86f05c33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index 79225b4..61cb77f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -209,9 +209,9 @@ Optional: | Property | Example | Description | |:---- |:---- | |`yarn.router.hostname` | `0.0.0.0` | Router host name. -|`yarn.router.clientrm.address` | `0.0.0.0:8032` | Router client address. | +|`yarn.router.clientrm.address` | `0.0.0.0:8050` | Router client address. | |`yarn.router.webapp.address` | `0.0.0.0:80` | Webapp address at the router. | -|`yarn.router.admin.address` | `0.0.0.0:8033` | Admin address at the router. | +|`yarn.router.admin.address` | `0.0.0.0:8052` | Admin address at the router. | |`yarn.router.webapp.https.address` | `0.0.0.0:443` | Secure webapp address at the router. | |`yarn.router.submit.retry` | `3` | The number of retries in the router before we give up. | |`yarn.federation.statestore.max-connections` | `10` | This is the maximum number of parallel connections each Router makes to the state-store. | @@ -227,7 +227,7 @@ These are extra configurations that should appear in the **conf/yarn-site.xml** |:---- |:---- | | `yarn.nodemanager.amrmproxy.enabled` | `true` | Whether or not the AMRMProxy is enabled. |`yarn.nodemanager.amrmproxy.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.nodemanager.amrmproxy.FederationInterceptor` | A comma-separated list of interceptors to be run at the amrmproxy. For federation the last step in the pipeline should be the FederationInterceptor. -| `yarn.client.failover.proxy-provider` | `org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider` | The class used to connect to the RMs by looking up the membership information in federation state-store. This must be set if federation is enabled, even if RM HA is not enabled.| +| `yarn.client.failover-proxy-provider` | `org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider` | The class used to connect to the RMs by looking up the membership information in federation state-store. This must be set if federation is enabled, even if RM HA is not enabled.| Optional: @@ -242,7 +242,7 @@ Currently, the only supported implementation of the state-store is Microsoft SQL Running a Sample Job -------------------- -In order to submit jobs to a Federation cluster one must create a seperate set of configs for the client from which said jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations: +In order to submit jobs to a Federation cluster one must create a seperate set of configs for the client from which jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations: | Property | Example | Description | |:--- |:--- | --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
