Repository: knox Updated Branches: refs/heads/master 10b1ed02e -> 165ea0fde
KNOX-1167: Support HDFS Federation in Knox topology generation Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/165ea0fd Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/165ea0fd Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/165ea0fd Branch: refs/heads/master Commit: 165ea0fde4df1c2cc0ba17f6f4a51e64ac61098c Parents: 10b1ed0 Author: Phil Zampino <[email protected]> Authored: Mon Feb 19 16:36:24 2018 -0500 Committer: Phil Zampino <[email protected]> Committed: Mon Feb 19 16:36:24 2018 -0500 ---------------------------------------------------------------------- .../discovery/ambari/AmbariCluster.java | 7 +- .../ambari/AmbariDynamicServiceURLCreator.java | 2 +- .../ambari/AmbariServiceDiscoveryMessages.java | 5 + .../discovery/ambari/NameNodeUrlCreator.java | 88 ++++ .../discovery/ambari/ServiceURLCreator.java | 6 +- .../discovery/ambari/ServiceURLFactory.java | 8 +- .../discovery/ambari/WebHdfsUrlCreator.java | 89 +++- .../ambari-service-discovery-url-mappings.xml | 22 - .../AmbariDynamicServiceURLCreatorTest.java | 417 +++++++++++++++---- .../simple/SimpleDescriptorHandler.java | 14 +- .../test/extension/DummyServiceDiscovery.java | 5 + .../PropertiesFileServiceDiscovery.java | 5 + .../topology/discovery/ServiceDiscovery.java | 10 + 13 files changed, 560 insertions(+), 118 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java index 2a1db09..35a1206 100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java @@ -128,8 +128,13 @@ class AmbariCluster implements ServiceDiscovery.Cluster { @Override public List<String> getServiceURLs(String serviceName) { + return getServiceURLs(serviceName, null); + } + + @Override + public List<String> getServiceURLs(String serviceName, Map<String, String> serviceParams) { List<String> urls = new ArrayList<>(); - urls.addAll(urlFactory.create(serviceName)); + urls.addAll(urlFactory.create(serviceName, serviceParams)); return urls; } http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java index dc4ac49..d769a18 100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java @@ -69,7 +69,7 @@ class AmbariDynamicServiceURLCreator implements ServiceURLCreator { config = new ServiceURLPropertyConfig(new ByteArrayInputStream(mappings.getBytes())); } - public List<String> create(String serviceName) { + public List<String> create(String serviceName, Map<String, String> serviceParams) { List<String> urls = new ArrayList<>(); Map<String, String> placeholderValues = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java index ffd04a9..258f4e1 100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java @@ -149,4 +149,9 @@ public interface AmbariServiceDiscoveryMessages { text = "Started Ambari cluster configuration monitor (checking every {0} seconds)") void startedAmbariConfigMonitor(final long pollingInterval); + + @Message(level = MessageLevel.WARN, + text = "The declared nameservice {0} is not defined in the HDFS configuration.") + void undefinedHDFSNameService(final String nameservice); + } http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/NameNodeUrlCreator.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/NameNodeUrlCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/NameNodeUrlCreator.java new file mode 100644 index 0000000..e387b67 --- /dev/null +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/NameNodeUrlCreator.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.knox.gateway.topology.discovery.ambari; + +import org.apache.knox.gateway.i18n.messages.MessagesFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + + +public class NameNodeUrlCreator implements ServiceURLCreator { + + private static final String SERVICE = "NAMENODE"; + + private static final String NAMESERVICE_PARAM = "discovery-nameservice"; + + private AmbariServiceDiscoveryMessages log = MessagesFactory.get(AmbariServiceDiscoveryMessages.class); + + private AmbariCluster cluster = null; + + NameNodeUrlCreator(AmbariCluster cluster) { + this.cluster = cluster; + } + + @Override + public List<String> create(String service, Map<String, String> serviceParams) { + List<String> urls = new ArrayList<>(); + + if (SERVICE.equals(service)) { + if (serviceParams != null && serviceParams.containsKey(NAMESERVICE_PARAM)) { + String declaredNameService = serviceParams.get(NAMESERVICE_PARAM); + + // Validate declared nameservice against available nameservices + if (!validateDeclaredNameService(cluster, declaredNameService)) { + log.undefinedHDFSNameService(declaredNameService); + } + + urls.add("hdfs://" + declaredNameService); + } else { + // Add the default nameservice URL to the result + AmbariCluster.ServiceConfiguration coreSite = cluster.getServiceConfiguration("HDFS", "core-site"); + if (coreSite != null) { + String defaultFS = coreSite.getProperties().get("fs.defaultFS"); + if (defaultFS != null) { + urls.add(defaultFS); + } + } + } + } + + return urls; + } + + // Verify whether the declared nameservice is among the configured nameservices in the cluster + private static boolean validateDeclaredNameService(AmbariCluster cluster, String declaredNameService) { + boolean isValid = false; + AmbariCluster.ServiceConfiguration hdfsSite = cluster.getServiceConfiguration("HDFS", "hdfs-site"); + if (hdfsSite != null) { + String nameservices = hdfsSite.getProperties().get("dfs.nameservices"); + if (nameservices != null) { + String[] namespaces = nameservices.split(","); + for (String ns : namespaces) { + if (ns.equals(declaredNameService)) { + isValid = true; + break; + } + } + } + } + return isValid; + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLCreator.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLCreator.java index c2a2d22..2984b44 100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLCreator.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLCreator.java @@ -17,16 +17,18 @@ package org.apache.knox.gateway.topology.discovery.ambari; import java.util.List; +import java.util.Map; public interface ServiceURLCreator { /** * Creates one or more cluster-specific URLs for the specified service. * - * @param service The service identifier. + * @param service The service identifier. + * @param serviceParams A map of parameters and their corresponding values for the specified service. * * @return A List of created URL strings; the list may be empty. */ - List<String> create(String service); + List<String> create(String service, Map<String, String> serviceParams); } http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLFactory.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLFactory.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLFactory.java index e009585..f114930 100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLFactory.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLFactory.java @@ -36,6 +36,7 @@ public class ServiceURLFactory { defaultURLCreator = new AmbariDynamicServiceURLCreator(cluster); // Custom (internal) URL creators + urlCreators.put("NAMENODE", new NameNodeUrlCreator(cluster)); urlCreators.put("WEBHDFS", new WebHdfsUrlCreator(cluster)); } @@ -55,11 +56,12 @@ public class ServiceURLFactory { /** * Create one or more cluster-specific URLs for the specified service. * - * @param service The service. + * @param service The service identifier. + * @param serviceParams A map of parameters and their corresponding values for the specified service. * * @return A List of service URL strings; the list may be empty. */ - public List<String> create(String service) { + public List<String> create(String service, Map<String, String> serviceParams) { List<String> urls = new ArrayList<>(); ServiceURLCreator creator = urlCreators.get(service); @@ -67,7 +69,7 @@ public class ServiceURLFactory { creator = defaultURLCreator; } - urls.addAll(creator.create(service)); + urls.addAll(creator.create(service, serviceParams)); return urls; } http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreator.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreator.java index 9c7d65b..6dcdf88 100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreator.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreator.java @@ -29,6 +29,8 @@ public class WebHdfsUrlCreator implements ServiceURLCreator { private static final String SERVICE = "WEBHDFS"; + private static final String NAMESERVICE_PARAM = "discovery-nameservice"; + private AmbariServiceDiscoveryMessages log = MessagesFactory.get(AmbariServiceDiscoveryMessages.class); private AmbariCluster cluster = null; @@ -38,7 +40,7 @@ public class WebHdfsUrlCreator implements ServiceURLCreator { } @Override - public List<String> create(String service) { + public List<String> create(String service, Map<String, String> serviceParams) { List<String> urls = new ArrayList<>(); if (SERVICE.equals(service)) { @@ -52,19 +54,62 @@ public class WebHdfsUrlCreator implements ServiceURLCreator { } if (nameServices != null && !nameServices.isEmpty()) { + String ns = null; + + // Parse the nameservices value + String[] namespaces = nameServices.split(","); + + if (namespaces.length > 1) { + String nsParam = (serviceParams != null) ? serviceParams.get(NAMESERVICE_PARAM) : null; + if (nsParam != null) { + if (!validateDeclaredNameService(sc, nsParam)) { + log.undefinedHDFSNameService(nsParam); + } + ns = nsParam; + } else { + // core-site.xml : dfs.defaultFS property (e.g., hdfs://ns1) + AmbariCluster.ServiceConfiguration coreSite = cluster.getServiceConfiguration("HDFS", "core-site"); + if (coreSite != null) { + String defaultFS = coreSite.getProperties().get("fs.defaultFS"); + if (defaultFS != null) { + ns = defaultFS.substring(defaultFS.lastIndexOf("/") + 1); + } + } + } + } + + // If only a single namespace, or no namespace specified and no default configured, use the first in the "list" + if (ns == null) { + ns = namespaces[0]; + } + // If it is an HA configuration Map<String, String> props = sc.getProperties(); - // Name node HTTP addresses are defined as properties of the form: - // dfs.namenode.http-address.<NAMESERVICES>.nn<INDEX> - // So, this iterates over the nn<INDEX> properties until there is no such property (since it cannot be known how - // many are defined by any other means). - int i = 1; - String propertyValue = getHANameNodeHttpAddress(props, nameServices, i++); - while (propertyValue != null) { - urls.add(createURL(propertyValue)); - propertyValue = getHANameNodeHttpAddress(props, nameServices, i++); + // More recent HDFS configurations support a property enumerating the node names associated with a + // nameservice. If this property is present, use its value to create the correct URLs. + String nameServiceNodes = props.get("dfs.ha.namenodes." + ns); + if (nameServiceNodes != null) { + String[] nodes = nameServiceNodes.split(","); + for (String node : nodes) { + String propertyValue = getHANameNodeHttpAddress(props, ns, node); + if (propertyValue != null) { + urls.add(createURL(propertyValue)); + } + } + } else { + // Name node HTTP addresses are defined as properties of the form: + // dfs.namenode.http-address.<NAMESERVICE>.nn<INDEX> + // So, this iterates over the nn<INDEX> properties until there is no such property (since it cannot be known how + // many are defined by any other means). + int i = 1; + String propertyValue = getHANameNodeHttpAddress(props, ns, i++); + while (propertyValue != null) { + urls.add(createURL(propertyValue)); + propertyValue = getHANameNodeHttpAddress(props, ns, i++); + } } + } else { // If it's not an HA configuration, get the single name node HTTP address urls.add(createURL(sc.getProperties().get("dfs.namenode.http-address"))); } @@ -74,8 +119,28 @@ public class WebHdfsUrlCreator implements ServiceURLCreator { return urls; } - private static String getHANameNodeHttpAddress(Map<String, String> props, String nameServices, int index) { - return props.get("dfs.namenode.http-address." + nameServices + ".nn" + index); + // Verify whether the declared nameservice is among the configured nameservices in the cluster + private static boolean validateDeclaredNameService(AmbariCluster.ServiceConfiguration hdfsSite, String declaredNameService) { + boolean isValid = false; + String nameservices = hdfsSite.getProperties().get("dfs.nameservices"); + if (nameservices != null) { + String[] namespaces = nameservices.split(","); + for (String ns : namespaces) { + if (ns.equals(declaredNameService)) { + isValid = true; + break; + } + } + } + return isValid; + } + + private static String getHANameNodeHttpAddress(Map<String, String> props, String nameService, int index) { + return props.get("dfs.namenode.http-address." + nameService + ".nn" + index); + } + + private static String getHANameNodeHttpAddress(Map<String, String> props, String nameService, String node) { + return props.get("dfs.namenode.http-address." + nameService + "." + node); } private static String createURL(String address) { http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml index 044fc45..3327ade 100644 --- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml +++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml @@ -23,28 +23,6 @@ <!-- ==================================================================== --> <service-discovery-url-mappings> - <service name="NAMENODE"> - <url-pattern>hdfs://{DFS_NAMENODE_ADDRESS}</url-pattern> - <properties> - <property name="DFS_NAMENODE_RPC_ADDRESS"> - <component>NAMENODE</component> - <config-property>dfs.namenode.rpc-address</config-property> - </property> - <property name="DFS_NAMESERVICES"> - <component>NAMENODE</component> - <config-property>dfs.nameservices</config-property> - </property> - <property name="DFS_NAMENODE_ADDRESS"> - <config-property> - <if property="DFS_NAMESERVICES"> - <then>DFS_NAMESERVICES</then> - <else>DFS_NAMENODE_RPC_ADDRESS</else> - </if> - </config-property> - </property> - </properties> - </service> - <service name="JOBTRACKER"> <url-pattern>rpc://{YARN_RM_ADDRESS}</url-pattern> <properties> http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java index c0b1de8..806d7c0 100644 --- a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java +++ b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java @@ -31,6 +31,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import static junit.framework.TestCase.assertNull; import static junit.framework.TestCase.assertTrue; import static junit.framework.TestCase.fail; import static org.junit.Assert.assertEquals; @@ -78,7 +79,7 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration); - List<String> urls = builder.create(SERVICE_NAME); + List<String> urls = builder.create(SERVICE_NAME, null); assertEquals(HOSTNAMES.length, urls.size()); validateServiceURLs(urls, HOSTNAMES, expectedScheme, HTTP_PORT, HTTP_PATH); @@ -93,7 +94,7 @@ public class AmbariDynamicServiceURLCreatorTest { EasyMock.replay(hiveServer); // Run the test - urls = builder.create(SERVICE_NAME); + urls = builder.create(SERVICE_NAME, null); assertEquals(HOSTNAMES.length, urls.size()); validateServiceURLs(urls, HOSTNAMES, expectedScheme, HTTP_PORT, ""); @@ -108,7 +109,7 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test expectedScheme = "https"; - urls = builder.create(SERVICE_NAME); + urls = builder.create(SERVICE_NAME, null); assertEquals(HOSTNAMES.length, urls.size()); validateServiceURLs(urls, HOSTNAMES, expectedScheme, HTTP_PORT, HTTP_PATH); } @@ -139,7 +140,7 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration); - String url = builder.create("RESOURCEMANAGER").get(0); + String url = builder.create("RESOURCEMANAGER", null).get(0); assertEquals("http://"; + HTTP_ADDRESS + "/ws", url); // HTTPS @@ -147,7 +148,7 @@ public class AmbariDynamicServiceURLCreatorTest { setResourceManagerComponentExpectations(resman, HTTP_ADDRESS, HTTPS_ADDRESS, "HTTPS_ONLY"); // Run the test - url = builder.create("RESOURCEMANAGER").get(0); + url = builder.create("RESOURCEMANAGER", null).get(0); assertEquals("https://"; + HTTPS_ADDRESS + "/ws", url); } @@ -184,63 +185,89 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration); - String url = builder.create("JOBTRACKER").get(0); + String url = builder.create("JOBTRACKER", null).get(0); assertEquals("rpc://" + ADDRESS, url); } @Test - public void testNameNodeURLFromInternalMapping() throws Exception { - testNameNodeURL(null); + public void testNameNodeURL() throws Exception { + final String ADDRESS = "host1:1234"; + String url = getTestNameNodeURL(ADDRESS); + assertEquals("hdfs://" + ADDRESS, url); } @Test - public void testNameNodeURLFromExternalMapping() throws Exception { - testNameNodeURL(TEST_MAPPING_CONFIG); + public void testNameNodeURLHADefaultNameService() throws Exception { + final String DEFAULT_NAMESERVICE = "ns1"; + String url = getTestNameNodeURL(DEFAULT_NAMESERVICE, "ns1"); + assertEquals("hdfs://" + DEFAULT_NAMESERVICE, url); } - private void testNameNodeURL(Object mappingConfiguration) throws Exception { - final String ADDRESS = "host1:1234"; + @Test + public void testNameNodeURLHADeclared() throws Exception { + final String delcaredNS = "ns1"; + Map<String, String> params = new HashMap<>(); + params.put("discovery-nameservice", delcaredNS); - AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class); - EasyMock.expect(namenode.getConfigProperty("dfs.namenode.rpc-address")).andReturn(ADDRESS).anyTimes(); - EasyMock.replay(namenode); + String url = getTestNameNodeURL("nn1", "ns1", params); + assertEquals("hdfs://" + delcaredNS, url); + } - AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class); - EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes(); - EasyMock.replay(cluster); + @Test + public void testNameNodeURLHADeclaredMultipleNameservices() throws Exception { + final String delcaredNS = "ns2"; + Map<String, String> params = new HashMap<>(); + params.put("discovery-nameservice", delcaredNS); - // Run the test - AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration); - String url = builder.create("NAMENODE").get(0); - assertEquals("hdfs://" + ADDRESS, url); + String url = getTestNameNodeURL("nn1", "ns1,ns2", params); + assertEquals("hdfs://" + delcaredNS, url); } - @Test - public void testNameNodeHAURLFromInternalMapping() throws Exception { - testNameNodeURLHA(null); + public void testNameNodeURLHADeclaredInvalid() throws Exception { + final String delcaredNS = "MyInvalidNameService"; + Map<String, String> params = new HashMap<>(); + params.put("discovery-nameservice", delcaredNS); + + String url = getTestNameNodeURL("nn1", "ns1,ns2", params); + assertEquals("Invalid nameservice declaration should still yield a URL.", + "hdfs://" + delcaredNS, + url); } - @Test - public void testNameNodeHAURLFromExternalMapping() throws Exception { - testNameNodeURLHA(TEST_MAPPING_CONFIG); + private String getTestNameNodeURL(String address) throws Exception { + return getTestNameNodeURL(address, null); } - private void testNameNodeURLHA(Object mappingConfiguration) throws Exception { - final String NAMESERVICE = "myNSCluster"; + private String getTestNameNodeURL(String defaultFSAddress, String nameservices) throws Exception { + return getTestNameNodeURL(defaultFSAddress, nameservices, Collections.emptyMap()); + } - AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class); - EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICE).anyTimes(); - EasyMock.replay(namenode); + private String getTestNameNodeURL(String defaultFSAddress, String nameservices, Map<String, String> serviceParams) throws Exception { + AmbariCluster.ServiceConfiguration coreSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class); + Map<String, String> coreProps = new HashMap<>(); + coreProps.put("fs.defaultFS", "hdfs://" + defaultFSAddress); + EasyMock.expect(coreSC.getProperties()).andReturn(coreProps).anyTimes(); + EasyMock.replay(coreSC); + + AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class); + Map<String, String> hdfsProps = new HashMap<>(); + if (nameservices != null) { + hdfsProps.put("dfs.nameservices", nameservices); + } + EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes(); + EasyMock.replay(hdfsSC); AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class); - EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes(); + EasyMock.expect(cluster.getServiceConfiguration("HDFS", "core-site")).andReturn(coreSC).anyTimes(); + EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes(); EasyMock.replay(cluster); - // Run the test - AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration); - String url = builder.create("NAMENODE").get(0); - assertEquals("hdfs://" + NAMESERVICE, url); + // Create the URL + List<String> urls = ServiceURLFactory.newInstance(cluster).create("NAMENODE", serviceParams); + assertNotNull(urls); + assertFalse(urls.isEmpty()); + return urls.get(0); } @@ -271,7 +298,7 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration); - String url = builder.create("WEBHCAT").get(0); + String url = builder.create("WEBHCAT", null).get(0); assertEquals("http://"; + HOSTNAME + ":" + PORT + "/templeton", url); } @@ -298,7 +325,7 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration); - String url = builder.create("OOZIE").get(0); + String url = builder.create("OOZIE", null).get(0); assertEquals(URL, url); } @@ -326,27 +353,18 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, mappingConfiguration); - List<String> urls = builder.create("WEBHBASE"); + List<String> urls = builder.create("WEBHBASE", null); validateServiceURLs(urls, HOSTNAMES, "http", "60080", null); } @Test - public void testWebHdfsURLFromInternalMapping() throws Exception { - testWebHdfsURL(null); - } - - @Test - public void testWebHdfsURLFromExternalMapping() throws Exception { - testWebHdfsURL(TEST_MAPPING_CONFIG); - } - - private void testWebHdfsURL(Object mappingConfiguration) throws Exception { + public void testWebHdfsURL() throws Exception { final String ADDRESS = "host3:1357"; - assertEquals("http://"; + ADDRESS + "/webhdfs", getTestWebHdfsURL(ADDRESS, mappingConfiguration)); + assertEquals("http://"; + ADDRESS + "/webhdfs", getTestWebHdfsURL(ADDRESS)); } - private String getTestWebHdfsURL(String address, Object mappingConfiguration) throws Exception { + private String getTestWebHdfsURL(String address) throws Exception { AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class); Map<String, String> hdfsProps = new HashMap<>(); hdfsProps.put("dfs.namenode.http-address", address); @@ -358,14 +376,14 @@ public class AmbariDynamicServiceURLCreatorTest { EasyMock.replay(cluster); // Create the URL - List<String> urls = ServiceURLFactory.newInstance(cluster).create("WEBHDFS"); + List<String> urls = ServiceURLFactory.newInstance(cluster).create("WEBHDFS", null); assertNotNull(urls); assertFalse(urls.isEmpty()); return urls.get(0); } @Test - public void testWebHdfsURLHA() throws Exception { + public void testWebHdfsURLHASingleNameService() throws Exception { final String NAMESERVICES = "myNameServicesCluster"; final String HTTP_ADDRESS_1 = "host1:50070"; final String HTTP_ADDRESS_2 = "host2:50077"; @@ -390,7 +408,262 @@ public class AmbariDynamicServiceURLCreatorTest { EasyMock.replay(cluster); // Create the URL - List<String> webhdfsURLs = ServiceURLFactory.newInstance(cluster).create("WEBHDFS"); + List<String> webhdfsURLs = ServiceURLFactory.newInstance(cluster).create("WEBHDFS", null); + assertEquals(2, webhdfsURLs.size()); + assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_1)); + assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_2)); + } + + + /** + * Test federated NameNode scenario, which chooses the "first" nameservice because there is no information from + * which one can be selected from among the set. + */ + @Test + public void testWebHdfsURLHAMultipleNameServicesNoDefaultFS() throws Exception { + final String NS1 = "myns1"; + final String NS2 = "myns2"; + final String NAMESERVICES = NS1 + "," + NS2; + final String HTTP_ADDRESS_11 = "host11:50070"; + final String HTTP_ADDRESS_12 = "host12:50077"; + final String HTTP_ADDRESS_21 = "host21:50070"; + final String HTTP_ADDRESS_22 = "host22:50077"; + + final String EXPECTED_ADDR_1 = "http://"; + HTTP_ADDRESS_11 + "/webhdfs"; + final String EXPECTED_ADDR_2 = "http://"; + HTTP_ADDRESS_12 + "/webhdfs"; + + AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class); + EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes(); + EasyMock.replay(namenode); + + AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class); + Map<String, String> hdfsProps = new HashMap<>(); + hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn1", HTTP_ADDRESS_11); + hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn2", HTTP_ADDRESS_12); + hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn1", HTTP_ADDRESS_21); + hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn2", HTTP_ADDRESS_22); + EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes(); + EasyMock.replay(hdfsSC); + + AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class); + EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes(); + EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes(); + EasyMock.replay(cluster); + + // Create the URL + List<String> webhdfsURLs = ServiceURLFactory.newInstance(cluster).create("WEBHDFS", null); + assertEquals(2, webhdfsURLs.size()); + assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_1)); + assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_2)); + } + + + /** + * Test federated NameNode scenario, relying on the core-site property for identifying the default nameservice. + */ + @Test + public void testWebHdfsURLFederatedNNWithDefaultFS() throws Exception { + final String NS1 = "myns1"; + final String NS2 = "myns2"; + final String NAMESERVICES = NS1 + "," + NS2; + final String HTTP_ADDRESS_11 = "host11:50070"; + final String HTTP_ADDRESS_12 = "host12:50077"; + final String HTTP_ADDRESS_21 = "host21:50070"; + final String HTTP_ADDRESS_22 = "host22:50077"; + + final String EXPECTED_ADDR_1 = "http://"; + HTTP_ADDRESS_21 + "/webhdfs"; + final String EXPECTED_ADDR_2 = "http://"; + HTTP_ADDRESS_22 + "/webhdfs"; + + AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class); + EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes(); + EasyMock.replay(namenode); + + AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class); + Map<String, String> hdfsProps = new HashMap<>(); + hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn1", HTTP_ADDRESS_11); + hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn2", HTTP_ADDRESS_12); + hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn1", HTTP_ADDRESS_21); + hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn2", HTTP_ADDRESS_22); + EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes(); + EasyMock.replay(hdfsSC); + + AmbariCluster.ServiceConfiguration coreSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class); + Map<String, String> coreProps = new HashMap<>(); + coreProps.put("fs.defaultFS", NS2); + EasyMock.expect(coreSC.getProperties()).andReturn(coreProps).anyTimes(); + EasyMock.replay(coreSC); + + AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class); + EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes(); + EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes(); + EasyMock.expect(cluster.getServiceConfiguration("HDFS", "core-site")).andReturn(coreSC).anyTimes(); + EasyMock.replay(cluster); + + // Create the URL + List<String> webhdfsURLs = ServiceURLFactory.newInstance(cluster).create("WEBHDFS", null); + assertEquals(2, webhdfsURLs.size()); + assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_1)); + assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_2)); + } + + + /** + * Recent version of HDFS config include properties for mapping NN nodes to nameservices (e.g., dfs.ha.namenode.ns1). + * This test verifies that discovery works correctly in those cases, when no nameservice is explicitly declared in + * a descriptor. + */ + @Test + public void testWebHdfsURLFederatedNNWithDefaultFSAndHaNodes() throws Exception { + final String NS1 = "myns1"; + final String NS2 = "myns2"; + final String NAMESERVICES = NS1 + "," + NS2; + final String HTTP_ADDRESS_11 = "host11:50070"; + final String HTTP_ADDRESS_12 = "host12:50077"; + final String HTTP_ADDRESS_21 = "host21:50070"; + final String HTTP_ADDRESS_22 = "host22:50077"; + + final String EXPECTED_ADDR_1 = "http://"; + HTTP_ADDRESS_21 + "/webhdfs"; + final String EXPECTED_ADDR_2 = "http://"; + HTTP_ADDRESS_22 + "/webhdfs"; + + AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class); + EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes(); + EasyMock.replay(namenode); + + AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class); + Map<String, String> hdfsProps = new HashMap<>(); + hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn11", HTTP_ADDRESS_11); + hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn12", HTTP_ADDRESS_12); + hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn21", HTTP_ADDRESS_21); + hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn22", HTTP_ADDRESS_22); + hdfsProps.put("dfs.ha.namenodes." + NS1, "nn11,nn12"); + hdfsProps.put("dfs.ha.namenodes." + NS2, "nn21,nn22"); + EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes(); + EasyMock.replay(hdfsSC); + + AmbariCluster.ServiceConfiguration coreSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class); + Map<String, String> coreProps = new HashMap<>(); + coreProps.put("fs.defaultFS", NS2); + EasyMock.expect(coreSC.getProperties()).andReturn(coreProps).anyTimes(); + EasyMock.replay(coreSC); + + AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class); + EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes(); + EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes(); + EasyMock.expect(cluster.getServiceConfiguration("HDFS", "core-site")).andReturn(coreSC).anyTimes(); + EasyMock.replay(cluster); + + // Create the URL + List<String> webhdfsURLs = ServiceURLFactory.newInstance(cluster).create("WEBHDFS", null); + assertEquals(2, webhdfsURLs.size()); + assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_1)); + assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_2)); + } + + + /** + * Recent version of HDFS config include properties for mapping NN nodes to nameservices (e.g., dfs.ha.namenode.ns1). + * This test verifies that discovery works correctly in those cases, when a nameservice is declared in descriptor. + */ + @Test + public void testWebHdfsURLFederatedNNDeclaredNS() throws Exception { + final String NS1 = "myns1"; + final String NS2 = "myns2"; + final String NAMESERVICES = NS1 + "," + NS2; + final String HTTP_ADDRESS_11 = "host11:50070"; + final String HTTP_ADDRESS_12 = "host12:50077"; + final String HTTP_ADDRESS_21 = "host21:50070"; + final String HTTP_ADDRESS_22 = "host22:50077"; + + final String EXPECTED_ADDR_1 = "http://"; + HTTP_ADDRESS_11 + "/webhdfs"; + final String EXPECTED_ADDR_2 = "http://"; + HTTP_ADDRESS_12 + "/webhdfs"; + + AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class); + EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes(); + EasyMock.replay(namenode); + + AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class); + Map<String, String> hdfsProps = new HashMap<>(); + hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn11", HTTP_ADDRESS_11); + hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn12", HTTP_ADDRESS_12); + hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn21", HTTP_ADDRESS_21); + hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn22", HTTP_ADDRESS_22); + hdfsProps.put("dfs.ha.namenodes." + NS1, "nn11,nn12"); + hdfsProps.put("dfs.ha.namenodes." + NS2, "nn21,nn22"); + hdfsProps.put("dfs.nameservices", NAMESERVICES); + EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes(); + EasyMock.replay(hdfsSC); + + AmbariCluster.ServiceConfiguration coreSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class); + Map<String, String> coreProps = new HashMap<>(); + coreProps.put("fs.defaultFS", NS2); + EasyMock.expect(coreSC.getProperties()).andReturn(coreProps).anyTimes(); + EasyMock.replay(coreSC); + + AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class); + EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes(); + EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes(); + EasyMock.expect(cluster.getServiceConfiguration("HDFS", "core-site")).andReturn(coreSC).anyTimes(); + EasyMock.replay(cluster); + + // Create the URL + Map<String, String> params = new HashMap<>(); + params.put("discovery-nameservice", NS1); // Declare the non-default nameservice + List<String> webhdfsURLs = ServiceURLFactory.newInstance(cluster).create("WEBHDFS", params); + + assertEquals(2, webhdfsURLs.size()); + assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_1)); + assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_2)); + } + + + /** + * Previous version of HDFS config DO NOT include properties for mapping NN nodes to nameservices. + * This test verifies that discovery works correctly in those cases, when a nameservice is declared in descriptor. + */ + @Test + public void testWebHdfsURLFederatedNNDeclaredNSWithoutHaNodes() throws Exception { + final String NS1 = "myns1"; + final String NS2 = "myns2"; + final String NAMESERVICES = NS1 + "," + NS2; + final String HTTP_ADDRESS_11 = "host11:50070"; + final String HTTP_ADDRESS_12 = "host12:50077"; + final String HTTP_ADDRESS_21 = "host21:50070"; + final String HTTP_ADDRESS_22 = "host22:50077"; + + final String EXPECTED_ADDR_1 = "http://"; + HTTP_ADDRESS_11 + "/webhdfs"; + final String EXPECTED_ADDR_2 = "http://"; + HTTP_ADDRESS_12 + "/webhdfs"; + + AmbariComponent namenode = EasyMock.createNiceMock(AmbariComponent.class); + EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes(); + EasyMock.replay(namenode); + + AmbariCluster.ServiceConfiguration hdfsSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class); + Map<String, String> hdfsProps = new HashMap<>(); + hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn1", HTTP_ADDRESS_11); + hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn2", HTTP_ADDRESS_12); + hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn1", HTTP_ADDRESS_21); + hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn2", HTTP_ADDRESS_22); + hdfsProps.put("dfs.nameservices", NAMESERVICES); + EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes(); + EasyMock.replay(hdfsSC); + + AmbariCluster.ServiceConfiguration coreSC = EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class); + Map<String, String> coreProps = new HashMap<>(); + coreProps.put("fs.defaultFS", NS2); + EasyMock.expect(coreSC.getProperties()).andReturn(coreProps).anyTimes(); + EasyMock.replay(coreSC); + + AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class); + EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes(); + EasyMock.expect(cluster.getServiceConfiguration("HDFS", "hdfs-site")).andReturn(hdfsSC).anyTimes(); + EasyMock.expect(cluster.getServiceConfiguration("HDFS", "core-site")).andReturn(coreSC).anyTimes(); + EasyMock.replay(cluster); + + // Create the URL + Map<String, String> params = new HashMap<>(); + params.put("discovery-nameservice", NS1); // Declare the non-default nameservice + List<String> webhdfsURLs = ServiceURLFactory.newInstance(cluster).create("WEBHDFS", params); + assertEquals(2, webhdfsURLs.size()); assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_1)); assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_2)); @@ -411,7 +684,7 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null); - List<String> urls = builder.create("ATLAS-API"); + List<String> urls = builder.create("ATLAS-API", null); assertEquals(1, urls.size()); assertEquals(ATLAS_REST_ADDRESS, urls.get(0)); } @@ -438,7 +711,7 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null); - List<String> urls = builder.create("ATLAS"); + List<String> urls = builder.create("ATLAS", null); validateServiceURLs(urls, HOSTNAMES, "http", HTTP_PORT, null); EasyMock.reset(atlasServer); @@ -449,7 +722,7 @@ public class AmbariDynamicServiceURLCreatorTest { EasyMock.replay(atlasServer); // Run the test - urls = builder.create("ATLAS"); + urls = builder.create("ATLAS", null); validateServiceURLs(urls, HOSTNAMES, "https", HTTPS_PORT, null); } @@ -476,7 +749,7 @@ public class AmbariDynamicServiceURLCreatorTest { AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null); // Run the test - validateServiceURLs(builder.create("ZEPPELIN"), HOSTNAMES, "http", HTTP_PORT, null); + validateServiceURLs(builder.create("ZEPPELIN", null), HOSTNAMES, "http", HTTP_PORT, null); EasyMock.reset(zeppelinMaster); EasyMock.expect(zeppelinMaster.getHostNames()).andReturn(atlastServerHosts).anyTimes(); @@ -486,7 +759,7 @@ public class AmbariDynamicServiceURLCreatorTest { EasyMock.replay(zeppelinMaster); // Run the test - validateServiceURLs(builder.create("ZEPPELIN"), HOSTNAMES, "https", HTTPS_PORT, null); + validateServiceURLs(builder.create("ZEPPELIN", null), HOSTNAMES, "https", HTTPS_PORT, null); } @@ -512,7 +785,7 @@ public class AmbariDynamicServiceURLCreatorTest { AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null); // Run the test - validateServiceURLs(builder.create("ZEPPELINUI"), HOSTNAMES, "http", HTTP_PORT, null); + validateServiceURLs(builder.create("ZEPPELINUI", null), HOSTNAMES, "http", HTTP_PORT, null); EasyMock.reset(zeppelinMaster); EasyMock.expect(zeppelinMaster.getHostNames()).andReturn(atlastServerHosts).anyTimes(); @@ -522,7 +795,7 @@ public class AmbariDynamicServiceURLCreatorTest { EasyMock.replay(zeppelinMaster); // Run the test - validateServiceURLs(builder.create("ZEPPELINUI"), HOSTNAMES, "https", HTTPS_PORT, null); + validateServiceURLs(builder.create("ZEPPELINUI", null), HOSTNAMES, "https", HTTPS_PORT, null); } @@ -548,7 +821,7 @@ public class AmbariDynamicServiceURLCreatorTest { AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null); // Run the test - validateServiceURLs(builder.create("ZEPPELINWS"), HOSTNAMES, "ws", HTTP_PORT, null); + validateServiceURLs(builder.create("ZEPPELINWS", null), HOSTNAMES, "ws", HTTP_PORT, null); EasyMock.reset(zeppelinMaster); EasyMock.expect(zeppelinMaster.getHostNames()).andReturn(atlastServerHosts).anyTimes(); @@ -558,7 +831,7 @@ public class AmbariDynamicServiceURLCreatorTest { EasyMock.replay(zeppelinMaster); // Run the test - validateServiceURLs(builder.create("ZEPPELINWS"), HOSTNAMES, "wss", HTTPS_PORT, null); + validateServiceURLs(builder.create("ZEPPELINWS", null), HOSTNAMES, "wss", HTTPS_PORT, null); } @@ -580,7 +853,7 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null); - List<String> urls = builder.create("DRUID-COORDINATOR"); + List<String> urls = builder.create("DRUID-COORDINATOR", null); validateServiceURLs(urls, HOSTNAMES, "http", PORT, null); } @@ -603,7 +876,7 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null); - List<String> urls = builder.create("DRUID-BROKER"); + List<String> urls = builder.create("DRUID-BROKER", null); validateServiceURLs(urls, HOSTNAMES, "http", PORT, null); } @@ -626,7 +899,7 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null); - List<String> urls = builder.create("DRUID-ROUTER"); + List<String> urls = builder.create("DRUID-ROUTER", null); validateServiceURLs(urls, HOSTNAMES, "http", PORT, null); } @@ -649,7 +922,7 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null); - List<String> urls = builder.create("DRUID-OVERLORD"); + List<String> urls = builder.create("DRUID-OVERLORD", null); validateServiceURLs(urls, HOSTNAMES, "http", PORT, null); } @@ -672,7 +945,7 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null); - List<String> urls = builder.create("SUPERSET"); + List<String> urls = builder.create("SUPERSET", null); validateServiceURLs(urls, HOSTNAMES, "http", PORT, null); } @@ -686,12 +959,12 @@ public class AmbariDynamicServiceURLCreatorTest { // Run the test AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null); - List<String> urls = builder.create("DRUID-BROKER"); + List<String> urls = builder.create("DRUID-BROKER", null); assertNotNull(urls); assertEquals(1, urls.size()); assertEquals("http://{HOST}:{PORT}";, urls.get(0)); - urls = builder.create("HIVE"); + urls = builder.create("HIVE", null); assertNotNull(urls); assertEquals(1, urls.size()); assertEquals("http://{HOST}:{PORT}/{PATH}";, urls.get(0)); http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java index 55411d8..01e6d43 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java @@ -76,6 +76,8 @@ public class SimpleDescriptorHandler { private static final SimpleDescriptorMessages log = MessagesFactory.get(SimpleDescriptorMessages.class); + private static final String DISCOVERY_PARAM_PREFIX = "discovery-"; + private static Map<String, ServiceDiscovery> discoveryInstances = new HashMap<>(); public static Map<String, File> handle(File desc) throws IOException { @@ -112,7 +114,7 @@ public class SimpleDescriptorHandler { List<String> descServiceURLs = descService.getURLs(); if (descServiceURLs == null || descServiceURLs.isEmpty()) { - descServiceURLs = cluster.getServiceURLs(serviceName); + descServiceURLs = cluster.getServiceURLs(serviceName, descService.getParams()); } // Validate the discovered service URLs @@ -440,10 +442,12 @@ public class SimpleDescriptorHandler { Map<String, String> svcParams = serviceParams.get(serviceName); if (svcParams != null) { for (String paramName : svcParams.keySet()) { - sw.write(" <param>\n"); - sw.write(" <name>" + paramName + "</name>\n"); - sw.write(" <value>" + svcParams.get(paramName) + "</value>\n"); - sw.write(" </param>\n"); + if (!(paramName.toLowerCase()).startsWith(DISCOVERY_PARAM_PREFIX)) { + sw.write(" <param>\n"); + sw.write(" <name>" + paramName + "</name>\n"); + sw.write(" <value>" + svcParams.get(paramName) + "</value>\n"); + sw.write(" </param>\n"); + } } } http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/DummyServiceDiscovery.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/DummyServiceDiscovery.java b/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/DummyServiceDiscovery.java index 21883d9..4ac88d3 100644 --- a/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/DummyServiceDiscovery.java +++ b/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/DummyServiceDiscovery.java @@ -44,6 +44,11 @@ public class DummyServiceDiscovery implements ServiceDiscovery { } @Override + public List<String> getServiceURLs(String serviceName, Map<String, String> serviceParams) { + return getServiceURLs(serviceName); + } + + @Override public ZooKeeperConfig getZooKeeperConfiguration(String serviceName) { return null; } http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/PropertiesFileServiceDiscovery.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/PropertiesFileServiceDiscovery.java b/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/PropertiesFileServiceDiscovery.java index 45307f1..1d89733 100644 --- a/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/PropertiesFileServiceDiscovery.java +++ b/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/PropertiesFileServiceDiscovery.java @@ -101,6 +101,11 @@ class PropertiesFileServiceDiscovery implements ServiceDiscovery { @Override public List<String> getServiceURLs(String serviceName) { + return getServiceURLs(serviceName, null); + } + + @Override + public List<String> getServiceURLs(String serviceName, Map<String, String> serviceParams) { return serviceURLS.get(serviceName); } http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java ---------------------------------------------------------------------- diff --git a/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java b/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java index b3366a3..01b2178 100644 --- a/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java +++ b/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java @@ -67,12 +67,22 @@ public interface ServiceDiscovery { /** * @param serviceName The name of the service + * * @return The URLs for the specified service in this cluster. */ List<String> getServiceURLs(String serviceName); /** + * @param serviceName The name of the service. + * @param serviceParams A map of parameters and their corresponding values for the specified service. + * + * @return The URLs for the specified service in this cluster. + */ + List<String> getServiceURLs(String serviceName, Map<String, String> serviceParams); + + /** * @param serviceName The name of the service + * * @return The HA configuration properties for the specified service in this cluster. */ ZooKeeperConfig getZooKeeperConfiguration(String serviceName);
