Repository: hadoop Updated Branches: refs/heads/trunk 6a76250b3 -> e27d5a13b
YARN-3714. AM proxy filter can not get RM webapp address from yarn.resourcemanager.hostname.rm-id. Contributed by Masatake Iwasaki Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e27d5a13 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e27d5a13 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e27d5a13 Branch: refs/heads/trunk Commit: e27d5a13b0623e3eb43ac773eccd082b9d6fa9d0 Parents: 6a76250 Author: Xuan <[email protected]> Authored: Tue Jun 16 14:05:09 2015 -0700 Committer: Xuan <[email protected]> Committed: Tue Jun 16 14:06:16 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../org/apache/hadoop/yarn/util/RMHAUtils.java | 33 ++++++++------ .../amfilter/TestAmFilterInitializer.java | 47 ++++++++++++++++++++ 3 files changed, 70 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e27d5a13/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index d0d1e06..b61eb27 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -527,6 +527,9 @@ Release 2.8.0 - UNRELEASED YARN-3794. TestRMEmbeddedElector fails because of ambiguous LOG reference. (Chengbing Liu via devaraj) + YARN-3714. AM proxy filter can not get RM webapp address from + yarn.resourcemanager.hostname.rm-id. (Masatake Iwasaki via xgong) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/e27d5a13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RMHAUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RMHAUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RMHAUtils.java index a7e1ce9..2e996e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RMHAUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RMHAUtils.java @@ -28,6 +28,7 @@ import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.HAServiceTarget; import org.apache.hadoop.yarn.client.RMHAServiceTarget; +import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; @Private @@ -71,25 +72,31 @@ public class RMHAUtils { public static List<String> getRMHAWebappAddresses( final YarnConfiguration conf) { + String prefix; + String defaultPort; + if (YarnConfiguration.useHttps(conf)) { + prefix = YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS; + defaultPort = ":" + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT; + } else { + prefix =YarnConfiguration.RM_WEBAPP_ADDRESS; + defaultPort = ":" + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT; + } Collection<String> rmIds = conf.getStringCollection(YarnConfiguration.RM_HA_IDS); List<String> addrs = new ArrayList<String>(); - if (YarnConfiguration.useHttps(conf)) { - for (String id : rmIds) { - String addr = conf.get( - YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + "." + id); - if (addr != null) { - addrs.add(addr); + for (String id : rmIds) { + String addr = conf.get(HAUtil.addSuffix(prefix, id)); + if (addr == null) { + String hostname = + conf.get(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, id)); + if (hostname != null) { + addr = hostname + defaultPort; } } - } else { - for (String id : rmIds) { - String addr = conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + id); - if (addr != null) { - addrs.add(addr); - } + if (addr != null) { + addrs.add(addr); } } return addrs; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/e27d5a13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java index bec62ce..63c2cf3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/amfilter/TestAmFilterInitializer.java @@ -135,6 +135,15 @@ public class TestAmFilterInitializer extends TestCase { assertEquals(WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf), proxyHosts.get(0)); + // Check conf in which only RM hostname is set + conf = new Configuration(false); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, + "${yarn.resourcemanager.hostname}:8088"); // default in yarn-default.xml + conf.set(YarnConfiguration.RM_HOSTNAME, "host1"); + proxyHosts = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf); + assertEquals(1, proxyHosts.size()); + assertEquals("host1:8088", proxyHosts.get(0)); + // Check PROXY_ADDRESS has priority conf = new Configuration(false); conf.set(YarnConfiguration.PROXY_ADDRESS, "host1:1000"); @@ -188,6 +197,44 @@ public class TestAmFilterInitializer extends TestCase { Collections.sort(proxyHosts); assertEquals("host5:5000", proxyHosts.get(0)); assertEquals("host6:6000", proxyHosts.get(1)); + + // Check config without explicit RM_WEBAPP_ADDRESS settings (RM HA) + conf = new Configuration(false); + conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2,rm3"); + conf.set(YarnConfiguration.RM_HOSTNAME + ".rm1", "host2"); + conf.set(YarnConfiguration.RM_HOSTNAME + ".rm2", "host3"); + conf.set(YarnConfiguration.RM_HOSTNAME + ".rm3", "host4"); + conf.set(YarnConfiguration.RM_HOSTNAME + ".rm4", "dummy"); + proxyHosts = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf); + assertEquals(3, proxyHosts.size()); + Collections.sort(proxyHosts); + assertEquals("host2:" + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT, + proxyHosts.get(0)); + assertEquals("host3:" + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT, + proxyHosts.get(1)); + assertEquals("host4:" + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT, + proxyHosts.get(2)); + + // Check config without explicit RM_WEBAPP_HTTPS_ADDRESS settings (RM HA) + conf = new Configuration(false); + conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, + HttpConfig.Policy.HTTPS_ONLY.toString()); + conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2,rm3"); + conf.set(YarnConfiguration.RM_HOSTNAME + ".rm1", "host2"); + conf.set(YarnConfiguration.RM_HOSTNAME + ".rm2", "host3"); + conf.set(YarnConfiguration.RM_HOSTNAME + ".rm3", "host4"); + conf.set(YarnConfiguration.RM_HOSTNAME + ".rm4", "dummy"); + proxyHosts = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf); + assertEquals(3, proxyHosts.size()); + Collections.sort(proxyHosts); + assertEquals("host2:" + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT, + proxyHosts.get(0)); + assertEquals("host3:" + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT, + proxyHosts.get(1)); + assertEquals("host4:" + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT, + proxyHosts.get(2)); } class MockAmFilterInitializer extends AmFilterInitializer {
