AMBARI-13693. Ambari Blueprint sets inconsistent Yarn RM values (DIPAYAN BHOWMICK via srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1e86d690 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1e86d690 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1e86d690 Branch: refs/heads/branch-dev-patch-upgrade Commit: 1e86d690d308451481e9043c393e60d3ba089f04 Parents: de3147d Author: Srimanth Gunturi <[email protected]> Authored: Tue Nov 3 10:09:32 2015 -0800 Committer: Srimanth Gunturi <[email protected]> Committed: Tue Nov 3 10:10:08 2015 -0800 ---------------------------------------------------------------------- .../ambari/view/utils/ambari/Services.java | 67 +++++++++++-------- .../ambari/view/utils/ambari/ServicesTest.java | 70 ++++++++++++++++++++ 2 files changed, 111 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1e86d690/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/ambari/Services.java ---------------------------------------------------------------------- diff --git a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/ambari/Services.java b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/ambari/Services.java index 421a365..8a50012 100644 --- a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/ambari/Services.java +++ b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/ambari/Services.java @@ -45,6 +45,11 @@ public class Services { private static final String YARN_RESOURCEMANAGER_HA_RM_IDS_KEY = "yarn.resourcemanager.ha.rm-ids"; private static final String YARN_RESOURCEMANAGER_HTTP_HA_PARTIAL_KEY = "yarn.resourcemanager.webapp.address."; private static final String YARN_RESOURCEMANAGER_HTTPS_HA_PARTIAL_KEY = "yarn.resourcemanager.webapp.https.address."; + private static final String YARN_RESOURCEMANAGER_HOSTNAME_KEY = "yarn.resourcemanager.hostname"; + private static final String YARN_RESOURCEMANAGER_HOSTNAME_PARTIAL_KEY = YARN_RESOURCEMANAGER_HOSTNAME_KEY + "."; + private static final String YARN_RESOURCEMANAGER_DEFAULT_HTTP_PORT = "8088"; + private static final String YARN_RESOURCEMANAGER_DEFAULT_HTTPS_PORT = "8090"; + private static final String YARN_ATS_URL = "yarn.timeline-server.url"; private final static String YARN_TIMELINE_WEBAPP_HTTP_ADDRESS_KEY = "yarn.timeline-service.webapp.address"; @@ -80,10 +85,9 @@ public class Services { private String getRMUrlFromClusterConfig() { String url; - String protocol; - String haEnabled = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_RESOURCEMANAGER_HA_ENABLED); - String httpPolicy = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_HTTP_POLICY); + String haEnabled = getYarnConfig(YARN_RESOURCEMANAGER_HA_ENABLED); + String httpPolicy = getYarnConfig(YARN_HTTP_POLICY); if (!(HTTP_ONLY.equals(httpPolicy) || HTTPS_ONLY.equals(httpPolicy))) { LOG.error(String.format("RA030 Unknown value %s of yarn-site/yarn.http.policy. HTTP_ONLY assumed.", httpPolicy)); @@ -94,34 +98,32 @@ public class Services { String[] urls = getRMHAUrls(httpPolicy); url = getActiveRMUrl(urls); } else { - if (httpPolicy.equals(HTTPS_ONLY)) { - protocol = "https"; - url = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_RESOURCEMANAGER_HTTPS_KEY); - } else { - protocol = "http"; - url = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_RESOURCEMANAGER_HTTP_KEY); + url = (httpPolicy.equals(HTTPS_ONLY)) ? getYarnConfig(YARN_RESOURCEMANAGER_HTTPS_KEY) + : getYarnConfig(YARN_RESOURCEMANAGER_HTTP_KEY); + + if (url == null || url.isEmpty()) { + url = getYarnConfig(YARN_RESOURCEMANAGER_HOSTNAME_KEY).trim() + ":" + getDefaultRMPort(httpPolicy); } - url = addProtocolIfMissing(url, protocol); + + url = addProtocolIfMissing(url, getProtocol(httpPolicy)); } return url; } private String[] getRMHAUrls(String httpPolicy) { - String haRmIds = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_RESOURCEMANAGER_HA_RM_IDS_KEY); + String haRmIds = getYarnConfig(YARN_RESOURCEMANAGER_HA_RM_IDS_KEY); String[] ids = haRmIds.split(","); int index = 0; String[] urls = new String[ids.length]; for (String id : ids) { - String url, protocol; - if (HTTPS_ONLY.equals(httpPolicy)) { - url = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_RESOURCEMANAGER_HTTPS_HA_PARTIAL_KEY + id); - protocol = "https"; - } else { - url = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_RESOURCEMANAGER_HTTP_HA_PARTIAL_KEY + id); - protocol = "http"; + String url = (HTTPS_ONLY.equals(httpPolicy)) ? getYarnConfig(YARN_RESOURCEMANAGER_HTTPS_HA_PARTIAL_KEY + id) + : getYarnConfig(YARN_RESOURCEMANAGER_HTTP_HA_PARTIAL_KEY + id); + + if (url == null || url.isEmpty()) { + url = getYarnConfig(YARN_RESOURCEMANAGER_HOSTNAME_PARTIAL_KEY + id).trim() + ":" + getDefaultRMPort(httpPolicy); } - urls[index++] = addProtocolIfMissing(url.trim(), protocol); + urls[index++] = addProtocolIfMissing(url.trim(), getProtocol(httpPolicy)); } return urls; } @@ -267,9 +269,8 @@ public class Services { private String getATSUrlFromCluster() { String url; - String protocol; - String httpPolicy = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_HTTP_POLICY); + String httpPolicy = getYarnConfig(YARN_HTTP_POLICY); if (!(HTTP_ONLY.equals(httpPolicy) || HTTPS_ONLY.equals(httpPolicy))) { LOG.error(String.format("RA030 Unknown value %s of yarn-site/yarn.http.policy. HTTP_ONLY assumed.", httpPolicy)); @@ -277,13 +278,11 @@ public class Services { } if (httpPolicy.equals(HTTPS_ONLY)) { - protocol = "https"; - url = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_TIMELINE_WEBAPP_HTTPS_ADDRESS_KEY); + url = getYarnConfig(YARN_TIMELINE_WEBAPP_HTTPS_ADDRESS_KEY); } else { - protocol = "http"; - url = ambariApi.getCluster().getConfigurationValue(YARN_SITE, YARN_TIMELINE_WEBAPP_HTTP_ADDRESS_KEY); + url = getYarnConfig(YARN_TIMELINE_WEBAPP_HTTP_ADDRESS_KEY); } - url = addProtocolIfMissing(url, protocol); + url = addProtocolIfMissing(url, getProtocol(httpPolicy)); return url; } @@ -317,4 +316,20 @@ public class Services { public static boolean hasProtocol(String url) { return url.matches("^[^:]+://.*$"); } + + private String getProtocol(String yarnHttpPolicyConfig) { + return HTTPS_ONLY.equals(yarnHttpPolicyConfig) ? "https" : "http"; + } + + private String getYarnConfig(String key) { + return ambariApi.getCluster().getConfigurationValue(YARN_SITE, key); + } + + /** + * @param yarnHttpPolicy - The HTTP Policy configured in YARN site file + * @return The default resource manager port depending on the http policy + */ + private String getDefaultRMPort(String yarnHttpPolicy) { + return (HTTPS_ONLY.equals(yarnHttpPolicy)) ? YARN_RESOURCEMANAGER_DEFAULT_HTTPS_PORT : YARN_RESOURCEMANAGER_DEFAULT_HTTP_PORT; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/1e86d690/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/ambari/ServicesTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/ambari/ServicesTest.java b/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/ambari/ServicesTest.java index 5d8918d..1cf5747 100644 --- a/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/ambari/ServicesTest.java +++ b/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/ambari/ServicesTest.java @@ -41,6 +41,8 @@ public class ServicesTest extends EasyMockSupport { private static final String HTTP_RM_URL2 = "http://c2.ambari.apache.org:8088";; private static final String HTTPS_RM_URL1 = "https://c1.ambari.apache.org:8088";; private static final String HTTPS_RM_URL2 = "https://c2.ambari.apache.org:8088";; + private static final String RM_URL1_HOST = "c1.ambari.apache.org"; + private static final String RM_URL2_HOST = "c2.ambari.apache.org"; private static final String RM_URL1_HOST_PORT = "c1.ambari.apache.org:8088"; private static final String RM_URL2_HOST_PORT = "c2.ambari.apache.org:8088"; private static final String RM_INFO_API_ENDPOINT = Services.RM_INFO_API_ENDPOINT; @@ -274,6 +276,59 @@ public class ServicesTest extends EasyMockSupport { } @Test + public void shouldConnectToDefaultHostPortInClusterModeWhenWebaddressConfigIsEmpty() throws Exception { + ViewContext viewContext = getViewContext(new HashMap<String, String>()); + AmbariApi ambariApi = createNiceMock(AmbariApi.class); + Cluster cluster = createNiceMock(Cluster.class); + + expect(ambariApi.isClusterAssociated()).andReturn(true).anyTimes(); + setClusterExpectationWithEmptyWebappConfig(cluster, "HTTP_ONLY"); + expect(ambariApi.getCluster()).andReturn(cluster).anyTimes(); + + Services services = new Services(ambariApi, viewContext); + + replayAll(); + assertEquals("http://"; + RM_URL1_HOST + ":8088", services.getRMUrl()); + + reset(cluster); + setClusterExpectationWithEmptyWebappConfig(cluster, "HTTPS_ONLY"); + replay(cluster); + + assertEquals("https://"; + RM_URL1_HOST + ":8090", services.getRMUrl()); + + } + + @Test + public void shouldConnectToDefaultHostPortInClusterModeWithHAWhenWebaddressConfigIsEmpty() throws Exception { + ViewContext viewContext = getViewContext(new HashMap<String, String>()); + AmbariApi ambariApi = createNiceMock(AmbariApi.class); + Cluster cluster = createNiceMock(Cluster.class); + URLStreamProvider urlStreamProvider = createNiceMock(URLStreamProvider.class); + + expect(ambariApi.isClusterAssociated()).andReturn(true).anyTimes(); + setClusterExpectationInHAWithEmptyWebappConfig(cluster, "HTTP_ONLY"); + expect(ambariApi.getCluster()).andReturn(cluster).anyTimes(); + + Services services = new Services(ambariApi, viewContext); + + InputStream inputStream = IOUtils.toInputStream("{\"clusterInfo\": {\"haState\": \"ACTIVE\"}}"); + expect(viewContext.getURLStreamProvider()).andReturn(urlStreamProvider).anyTimes(); + expect(urlStreamProvider.readFrom(eq("http://"; + RM_URL1_HOST + ":8088" + RM_INFO_API_ENDPOINT), eq("GET"), anyString(), EasyMock.<Map<String, String>>anyObject())).andReturn(inputStream); + + replayAll(); + assertEquals("http://"; + RM_URL1_HOST + ":8088", services.getRMUrl()); + + reset(cluster, urlStreamProvider); + setClusterExpectationInHAWithEmptyWebappConfig(cluster, "HTTPS_ONLY"); + inputStream = IOUtils.toInputStream("{\"clusterInfo\": {\"haState\": \"ACTIVE\"}}"); + expect(urlStreamProvider.readFrom(eq("https://"; + RM_URL1_HOST + ":8090" + RM_INFO_API_ENDPOINT), eq("GET"), anyString(), EasyMock.<Map<String, String>>anyObject())).andReturn(inputStream); + replay(cluster, urlStreamProvider); + + assertEquals("https://"; + RM_URL1_HOST + ":8090", services.getRMUrl()); + + } + + @Test public void shouldFetchRMUrlsWhileHAEnabledInClusterMode() throws Exception { ViewContext viewContext = getViewContext(new HashMap<String, String>()); AmbariApi ambariApi = createNiceMock(AmbariApi.class); @@ -341,6 +396,21 @@ public class ServicesTest extends EasyMockSupport { expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.webapp.https.address.rm2")).andReturn(RM_URL2_HOST_PORT); } + private void setClusterExpectationInHAWithEmptyWebappConfig(Cluster cluster, String httpPolicy) { + expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.ha.enabled")).andReturn("true"); + expect(cluster.getConfigurationValue("yarn-site", "yarn.http.policy")).andReturn(httpPolicy); + expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.ha.rm-ids")).andReturn("rm1,rm2"); + expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.hostname.rm1")).andReturn(RM_URL1_HOST); + expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.hostname.rm2")).andReturn(RM_URL2_HOST); + + } + + private void setClusterExpectationWithEmptyWebappConfig(Cluster cluster, String httpPolicy) { + expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.ha.enabled")).andReturn("false"); + expect(cluster.getConfigurationValue("yarn-site", "yarn.http.policy")).andReturn(httpPolicy); + expect(cluster.getConfigurationValue("yarn-site", "yarn.resourcemanager.hostname")).andReturn(RM_URL1_HOST); + } + private ViewContext getViewContext(Map<String, String> map) { ViewContext viewContextMock = createNiceMock(ViewContext.class); expect(viewContextMock.getProperties()).andReturn(map);
