Repository: knox
Updated Branches:
  refs/heads/master 10b1ed02e -> 165ea0fde


KNOX-1167: Support HDFS Federation in Knox topology generation


Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/165ea0fd
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/165ea0fd
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/165ea0fd

Branch: refs/heads/master
Commit: 165ea0fde4df1c2cc0ba17f6f4a51e64ac61098c
Parents: 10b1ed0
Author: Phil Zampino <pzamp...@apache.org>
Authored: Mon Feb 19 16:36:24 2018 -0500
Committer: Phil Zampino <pzamp...@apache.org>
Committed: Mon Feb 19 16:36:24 2018 -0500

----------------------------------------------------------------------
 .../discovery/ambari/AmbariCluster.java         |   7 +-
 .../ambari/AmbariDynamicServiceURLCreator.java  |   2 +-
 .../ambari/AmbariServiceDiscoveryMessages.java  |   5 +
 .../discovery/ambari/NameNodeUrlCreator.java    |  88 ++++
 .../discovery/ambari/ServiceURLCreator.java     |   6 +-
 .../discovery/ambari/ServiceURLFactory.java     |   8 +-
 .../discovery/ambari/WebHdfsUrlCreator.java     |  89 +++-
 .../ambari-service-discovery-url-mappings.xml   |  22 -
 .../AmbariDynamicServiceURLCreatorTest.java     | 417 +++++++++++++++----
 .../simple/SimpleDescriptorHandler.java         |  14 +-
 .../test/extension/DummyServiceDiscovery.java   |   5 +
 .../PropertiesFileServiceDiscovery.java         |   5 +
 .../topology/discovery/ServiceDiscovery.java    |  10 +
 13 files changed, 560 insertions(+), 118 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java
----------------------------------------------------------------------
diff --git 
a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java
 
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java
index 2a1db09..35a1206 100644
--- 
a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java
+++ 
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java
@@ -128,8 +128,13 @@ class AmbariCluster implements ServiceDiscovery.Cluster {
 
     @Override
     public List<String> getServiceURLs(String serviceName) {
+        return getServiceURLs(serviceName, null);
+    }
+
+    @Override
+    public List<String> getServiceURLs(String serviceName, Map<String, String> 
serviceParams) {
         List<String> urls = new ArrayList<>();
-        urls.addAll(urlFactory.create(serviceName));
+        urls.addAll(urlFactory.create(serviceName, serviceParams));
         return urls;
     }
 

http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java
----------------------------------------------------------------------
diff --git 
a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java
 
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java
index dc4ac49..d769a18 100644
--- 
a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java
+++ 
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java
@@ -69,7 +69,7 @@ class AmbariDynamicServiceURLCreator implements 
ServiceURLCreator {
         config = new ServiceURLPropertyConfig(new 
ByteArrayInputStream(mappings.getBytes()));
     }
 
-    public List<String> create(String serviceName) {
+    public List<String> create(String serviceName, Map<String, String> 
serviceParams) {
         List<String> urls = new ArrayList<>();
 
         Map<String, String> placeholderValues = new HashMap<>();

http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java
----------------------------------------------------------------------
diff --git 
a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java
 
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java
index ffd04a9..258f4e1 100644
--- 
a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java
+++ 
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java
@@ -149,4 +149,9 @@ public interface AmbariServiceDiscoveryMessages {
              text = "Started Ambari cluster configuration monitor (checking 
every {0} seconds)")
     void startedAmbariConfigMonitor(final long pollingInterval);
 
+
+    @Message(level = MessageLevel.WARN,
+             text = "The declared nameservice {0} is not defined in the HDFS 
configuration.")
+    void undefinedHDFSNameService(final String nameservice);
+
 }

http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/NameNodeUrlCreator.java
----------------------------------------------------------------------
diff --git 
a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/NameNodeUrlCreator.java
 
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/NameNodeUrlCreator.java
new file mode 100644
index 0000000..e387b67
--- /dev/null
+++ 
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/NameNodeUrlCreator.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations 
under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.discovery.ambari;
+
+import org.apache.knox.gateway.i18n.messages.MessagesFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+
+public class NameNodeUrlCreator implements ServiceURLCreator {
+
+  private static final String SERVICE = "NAMENODE";
+
+  private static final String NAMESERVICE_PARAM = "discovery-nameservice";
+
+  private AmbariServiceDiscoveryMessages log = 
MessagesFactory.get(AmbariServiceDiscoveryMessages.class);
+
+  private AmbariCluster cluster = null;
+
+  NameNodeUrlCreator(AmbariCluster cluster) {
+    this.cluster = cluster;
+  }
+
+  @Override
+  public List<String> create(String service, Map<String, String> 
serviceParams) {
+    List<String> urls = new ArrayList<>();
+
+    if (SERVICE.equals(service)) {
+      if (serviceParams != null && 
serviceParams.containsKey(NAMESERVICE_PARAM)) {
+        String declaredNameService = serviceParams.get(NAMESERVICE_PARAM);
+
+        // Validate declared nameservice against available nameservices
+        if (!validateDeclaredNameService(cluster, declaredNameService)) {
+          log.undefinedHDFSNameService(declaredNameService);
+        }
+
+        urls.add("hdfs://" + declaredNameService);
+      } else {
+        // Add the default nameservice URL to the result
+        AmbariCluster.ServiceConfiguration coreSite = 
cluster.getServiceConfiguration("HDFS", "core-site");
+        if (coreSite != null) {
+          String defaultFS = coreSite.getProperties().get("fs.defaultFS");
+          if (defaultFS != null) {
+            urls.add(defaultFS);
+          }
+        }
+      }
+    }
+
+    return urls;
+  }
+
+  // Verify whether the declared nameservice is among the configured 
nameservices in the cluster
+  private static boolean validateDeclaredNameService(AmbariCluster cluster, 
String declaredNameService) {
+    boolean isValid = false;
+    AmbariCluster.ServiceConfiguration hdfsSite = 
cluster.getServiceConfiguration("HDFS", "hdfs-site");
+    if (hdfsSite != null) {
+      String nameservices = hdfsSite.getProperties().get("dfs.nameservices");
+      if (nameservices != null) {
+        String[] namespaces = nameservices.split(",");
+        for (String ns : namespaces) {
+          if (ns.equals(declaredNameService)) {
+            isValid = true;
+            break;
+          }
+        }
+      }
+    }
+    return isValid;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLCreator.java
----------------------------------------------------------------------
diff --git 
a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLCreator.java
 
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLCreator.java
index c2a2d22..2984b44 100644
--- 
a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLCreator.java
+++ 
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLCreator.java
@@ -17,16 +17,18 @@
 package org.apache.knox.gateway.topology.discovery.ambari;
 
 import java.util.List;
+import java.util.Map;
 
 public interface ServiceURLCreator {
 
   /**
    * Creates one or more cluster-specific URLs for the specified service.
    *
-   * @param service The service identifier.
+   * @param service       The service identifier.
+   * @param serviceParams A map of parameters and their corresponding values 
for the specified service.
    *
    * @return A List of created URL strings; the list may be empty.
    */
-  List<String> create(String service);
+  List<String> create(String service, Map<String, String> serviceParams);
 
 }

http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLFactory.java
----------------------------------------------------------------------
diff --git 
a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLFactory.java
 
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLFactory.java
index e009585..f114930 100644
--- 
a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLFactory.java
+++ 
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLFactory.java
@@ -36,6 +36,7 @@ public class ServiceURLFactory {
     defaultURLCreator = new AmbariDynamicServiceURLCreator(cluster);
 
     // Custom (internal) URL creators
+    urlCreators.put("NAMENODE", new NameNodeUrlCreator(cluster));
     urlCreators.put("WEBHDFS", new WebHdfsUrlCreator(cluster));
   }
 
@@ -55,11 +56,12 @@ public class ServiceURLFactory {
   /**
    * Create one or more cluster-specific URLs for the specified service.
    *
-   * @param service The service.
+   * @param service       The service identifier.
+   * @param serviceParams A map of parameters and their corresponding values 
for the specified service.
    *
    * @return A List of service URL strings; the list may be empty.
    */
-  public List<String> create(String service) {
+  public List<String> create(String service, Map<String, String> 
serviceParams) {
     List<String> urls = new ArrayList<>();
 
     ServiceURLCreator creator = urlCreators.get(service);
@@ -67,7 +69,7 @@ public class ServiceURLFactory {
       creator = defaultURLCreator;
     }
 
-    urls.addAll(creator.create(service));
+    urls.addAll(creator.create(service, serviceParams));
 
     return urls;
   }

http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreator.java
----------------------------------------------------------------------
diff --git 
a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreator.java
 
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreator.java
index 9c7d65b..6dcdf88 100644
--- 
a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreator.java
+++ 
b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/WebHdfsUrlCreator.java
@@ -29,6 +29,8 @@ public class WebHdfsUrlCreator implements ServiceURLCreator {
 
   private static final String SERVICE = "WEBHDFS";
 
+  private static final String NAMESERVICE_PARAM = "discovery-nameservice";
+
   private AmbariServiceDiscoveryMessages log = 
MessagesFactory.get(AmbariServiceDiscoveryMessages.class);
 
   private AmbariCluster cluster = null;
@@ -38,7 +40,7 @@ public class WebHdfsUrlCreator implements ServiceURLCreator {
   }
 
   @Override
-  public List<String> create(String service) {
+  public List<String> create(String service, Map<String, String> 
serviceParams) {
     List<String> urls = new ArrayList<>();
 
     if (SERVICE.equals(service)) {
@@ -52,19 +54,62 @@ public class WebHdfsUrlCreator implements ServiceURLCreator 
{
         }
 
         if (nameServices != null && !nameServices.isEmpty()) {
+          String ns = null;
+
+          // Parse the nameservices value
+          String[] namespaces = nameServices.split(",");
+
+          if (namespaces.length > 1) {
+            String nsParam = (serviceParams != null) ? 
serviceParams.get(NAMESERVICE_PARAM) : null;
+            if (nsParam != null) {
+              if (!validateDeclaredNameService(sc, nsParam)) {
+                log.undefinedHDFSNameService(nsParam);
+              }
+              ns = nsParam;
+            } else {
+              // core-site.xml : dfs.defaultFS property (e.g., hdfs://ns1)
+              AmbariCluster.ServiceConfiguration coreSite = 
cluster.getServiceConfiguration("HDFS", "core-site");
+              if (coreSite != null) {
+                String defaultFS = 
coreSite.getProperties().get("fs.defaultFS");
+                if (defaultFS != null) {
+                  ns = defaultFS.substring(defaultFS.lastIndexOf("/") + 1);
+                }
+              }
+            }
+          }
+
+          // If only a single namespace, or no namespace specified and no 
default configured, use the first in the "list"
+          if (ns == null) {
+             ns = namespaces[0];
+          }
+
           // If it is an HA configuration
           Map<String, String> props = sc.getProperties();
 
-          // Name node HTTP addresses are defined as properties of the form:
-          //      dfs.namenode.http-address.<NAMESERVICES>.nn<INDEX>
-          // So, this iterates over the nn<INDEX> properties until there is no 
such property (since it cannot be known how
-          // many are defined by any other means).
-          int i = 1;
-          String propertyValue = getHANameNodeHttpAddress(props, nameServices, 
i++);
-          while (propertyValue != null) {
-            urls.add(createURL(propertyValue));
-            propertyValue = getHANameNodeHttpAddress(props, nameServices, i++);
+          // More recent HDFS configurations support a property enumerating 
the node names associated with a
+          // nameservice. If this property is present, use its value to create 
the correct URLs.
+          String nameServiceNodes = props.get("dfs.ha.namenodes." + ns);
+          if (nameServiceNodes != null) {
+            String[] nodes = nameServiceNodes.split(",");
+            for (String node : nodes) {
+              String propertyValue = getHANameNodeHttpAddress(props, ns, node);
+              if (propertyValue != null) {
+                urls.add(createURL(propertyValue));
+              }
+            }
+          } else {
+            // Name node HTTP addresses are defined as properties of the form:
+            //      dfs.namenode.http-address.<NAMESERVICE>.nn<INDEX>
+            // So, this iterates over the nn<INDEX> properties until there is 
no such property (since it cannot be known how
+            // many are defined by any other means).
+            int i = 1;
+            String propertyValue = getHANameNodeHttpAddress(props, ns, i++);
+            while (propertyValue != null) {
+              urls.add(createURL(propertyValue));
+              propertyValue = getHANameNodeHttpAddress(props, ns, i++);
+            }
           }
+
         } else { // If it's not an HA configuration, get the single name node 
HTTP address
           
urls.add(createURL(sc.getProperties().get("dfs.namenode.http-address")));
         }
@@ -74,8 +119,28 @@ public class WebHdfsUrlCreator implements ServiceURLCreator 
{
     return urls;
   }
 
-  private static String getHANameNodeHttpAddress(Map<String, String> props, 
String nameServices, int index) {
-    return props.get("dfs.namenode.http-address." + nameServices + ".nn" + 
index);
+  // Verify whether the declared nameservice is among the configured 
nameservices in the cluster
+  private static boolean 
validateDeclaredNameService(AmbariCluster.ServiceConfiguration hdfsSite, String 
declaredNameService) {
+    boolean isValid = false;
+    String nameservices = hdfsSite.getProperties().get("dfs.nameservices");
+    if (nameservices != null) {
+      String[] namespaces = nameservices.split(",");
+      for (String ns : namespaces) {
+        if (ns.equals(declaredNameService)) {
+          isValid = true;
+          break;
+        }
+      }
+    }
+    return isValid;
+  }
+
+  private static String getHANameNodeHttpAddress(Map<String, String> props, 
String nameService, int index) {
+    return props.get("dfs.namenode.http-address." + nameService + ".nn" + 
index);
+  }
+
+  private static String getHANameNodeHttpAddress(Map<String, String> props, 
String nameService, String node) {
+    return props.get("dfs.namenode.http-address." + nameService + "." + node);
   }
 
   private static String createURL(String address) {

http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
----------------------------------------------------------------------
diff --git 
a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
 
b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
index 044fc45..3327ade 100644
--- 
a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
+++ 
b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml
@@ -23,28 +23,6 @@
 <!-- ==================================================================== -->
 <service-discovery-url-mappings>
 
-    <service name="NAMENODE">
-        <url-pattern>hdfs://{DFS_NAMENODE_ADDRESS}</url-pattern>
-        <properties>
-            <property name="DFS_NAMENODE_RPC_ADDRESS">
-                <component>NAMENODE</component>
-                <config-property>dfs.namenode.rpc-address</config-property>
-            </property>
-            <property name="DFS_NAMESERVICES">
-                <component>NAMENODE</component>
-                <config-property>dfs.nameservices</config-property>
-            </property>
-            <property name="DFS_NAMENODE_ADDRESS">
-                <config-property>
-                    <if property="DFS_NAMESERVICES">
-                        <then>DFS_NAMESERVICES</then>
-                        <else>DFS_NAMENODE_RPC_ADDRESS</else>
-                    </if>
-                </config-property>
-            </property>
-        </properties>
-    </service>
-
     <service name="JOBTRACKER">
         <url-pattern>rpc://{YARN_RM_ADDRESS}</url-pattern>
         <properties>

http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
----------------------------------------------------------------------
diff --git 
a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
 
b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
index c0b1de8..806d7c0 100644
--- 
a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
+++ 
b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreatorTest.java
@@ -31,6 +31,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
+import static junit.framework.TestCase.assertNull;
 import static junit.framework.TestCase.assertTrue;
 import static junit.framework.TestCase.fail;
 import static org.junit.Assert.assertEquals;
@@ -78,7 +79,7 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, 
mappingConfiguration);
-        List<String> urls = builder.create(SERVICE_NAME);
+        List<String> urls = builder.create(SERVICE_NAME, null);
         assertEquals(HOSTNAMES.length, urls.size());
         validateServiceURLs(urls, HOSTNAMES, expectedScheme, HTTP_PORT, 
HTTP_PATH);
 
@@ -93,7 +94,7 @@ public class AmbariDynamicServiceURLCreatorTest {
         EasyMock.replay(hiveServer);
 
         // Run the test
-        urls = builder.create(SERVICE_NAME);
+        urls = builder.create(SERVICE_NAME, null);
         assertEquals(HOSTNAMES.length, urls.size());
         validateServiceURLs(urls, HOSTNAMES, expectedScheme, HTTP_PORT, "");
 
@@ -108,7 +109,7 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         expectedScheme = "https";
-        urls = builder.create(SERVICE_NAME);
+        urls = builder.create(SERVICE_NAME, null);
         assertEquals(HOSTNAMES.length, urls.size());
         validateServiceURLs(urls, HOSTNAMES, expectedScheme, HTTP_PORT, 
HTTP_PATH);
     }
@@ -139,7 +140,7 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, 
mappingConfiguration);
-        String url = builder.create("RESOURCEMANAGER").get(0);
+        String url = builder.create("RESOURCEMANAGER", null).get(0);
         assertEquals("http://"; + HTTP_ADDRESS + "/ws", url);
 
         // HTTPS
@@ -147,7 +148,7 @@ public class AmbariDynamicServiceURLCreatorTest {
         setResourceManagerComponentExpectations(resman, HTTP_ADDRESS, 
HTTPS_ADDRESS, "HTTPS_ONLY");
 
         // Run the test
-        url = builder.create("RESOURCEMANAGER").get(0);
+        url = builder.create("RESOURCEMANAGER", null).get(0);
         assertEquals("https://"; + HTTPS_ADDRESS + "/ws", url);
     }
 
@@ -184,63 +185,89 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, 
mappingConfiguration);
-        String url = builder.create("JOBTRACKER").get(0);
+        String url = builder.create("JOBTRACKER", null).get(0);
         assertEquals("rpc://" + ADDRESS, url);
     }
 
     @Test
-    public void testNameNodeURLFromInternalMapping() throws Exception {
-        testNameNodeURL(null);
+    public void testNameNodeURL() throws Exception {
+        final String ADDRESS = "host1:1234";
+        String url = getTestNameNodeURL(ADDRESS);
+        assertEquals("hdfs://" + ADDRESS, url);
     }
 
     @Test
-    public void testNameNodeURLFromExternalMapping() throws Exception {
-        testNameNodeURL(TEST_MAPPING_CONFIG);
+    public void testNameNodeURLHADefaultNameService() throws Exception {
+        final String DEFAULT_NAMESERVICE = "ns1";
+        String url = getTestNameNodeURL(DEFAULT_NAMESERVICE, "ns1");
+        assertEquals("hdfs://" + DEFAULT_NAMESERVICE, url);
     }
 
-    private void testNameNodeURL(Object mappingConfiguration) throws Exception 
{
-        final String ADDRESS = "host1:1234";
+    @Test
+    public void testNameNodeURLHADeclared() throws Exception {
+        final String delcaredNS = "ns1";
+        Map<String, String> params = new HashMap<>();
+        params.put("discovery-nameservice", delcaredNS);
 
-        AmbariComponent namenode = 
EasyMock.createNiceMock(AmbariComponent.class);
-        
EasyMock.expect(namenode.getConfigProperty("dfs.namenode.rpc-address")).andReturn(ADDRESS).anyTimes();
-        EasyMock.replay(namenode);
+        String url = getTestNameNodeURL("nn1", "ns1", params);
+        assertEquals("hdfs://" + delcaredNS, url);
+    }
 
-        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
-        
EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
-        EasyMock.replay(cluster);
+    @Test
+    public void testNameNodeURLHADeclaredMultipleNameservices() throws 
Exception {
+        final String delcaredNS = "ns2";
+        Map<String, String> params = new HashMap<>();
+        params.put("discovery-nameservice", delcaredNS);
 
-        // Run the test
-        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, 
mappingConfiguration);
-        String url = builder.create("NAMENODE").get(0);
-        assertEquals("hdfs://" + ADDRESS, url);
+        String url = getTestNameNodeURL("nn1", "ns1,ns2", params);
+        assertEquals("hdfs://" + delcaredNS, url);
     }
 
-
     @Test
-    public void testNameNodeHAURLFromInternalMapping() throws Exception {
-        testNameNodeURLHA(null);
+    public void testNameNodeURLHADeclaredInvalid() throws Exception {
+        final String delcaredNS = "MyInvalidNameService";
+        Map<String, String> params = new HashMap<>();
+        params.put("discovery-nameservice", delcaredNS);
+
+        String url = getTestNameNodeURL("nn1", "ns1,ns2", params);
+        assertEquals("Invalid nameservice declaration should still yield a 
URL.",
+                     "hdfs://" + delcaredNS,
+                     url);
     }
 
-    @Test
-    public void testNameNodeHAURLFromExternalMapping() throws Exception {
-        testNameNodeURLHA(TEST_MAPPING_CONFIG);
+    private String getTestNameNodeURL(String address) throws Exception {
+        return getTestNameNodeURL(address, null);
     }
 
-    private void testNameNodeURLHA(Object mappingConfiguration) throws 
Exception {
-        final String NAMESERVICE = "myNSCluster";
+    private String getTestNameNodeURL(String defaultFSAddress, String 
nameservices) throws Exception {
+        return getTestNameNodeURL(defaultFSAddress, nameservices, 
Collections.emptyMap());
+    }
 
-        AmbariComponent namenode = 
EasyMock.createNiceMock(AmbariComponent.class);
-        
EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICE).anyTimes();
-        EasyMock.replay(namenode);
+    private String getTestNameNodeURL(String defaultFSAddress, String 
nameservices, Map<String, String> serviceParams) throws Exception {
+        AmbariCluster.ServiceConfiguration coreSC = 
EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+        Map<String, String> coreProps = new HashMap<>();
+        coreProps.put("fs.defaultFS", "hdfs://" + defaultFSAddress);
+        
EasyMock.expect(coreSC.getProperties()).andReturn(coreProps).anyTimes();
+        EasyMock.replay(coreSC);
+
+        AmbariCluster.ServiceConfiguration hdfsSC = 
EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+        Map<String, String> hdfsProps = new HashMap<>();
+        if (nameservices != null) {
+            hdfsProps.put("dfs.nameservices", nameservices);
+        }
+        
EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes();
+        EasyMock.replay(hdfsSC);
 
         AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
-        
EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
+        EasyMock.expect(cluster.getServiceConfiguration("HDFS", 
"core-site")).andReturn(coreSC).anyTimes();
+        EasyMock.expect(cluster.getServiceConfiguration("HDFS", 
"hdfs-site")).andReturn(hdfsSC).anyTimes();
         EasyMock.replay(cluster);
 
-        // Run the test
-        AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, 
mappingConfiguration);
-        String url = builder.create("NAMENODE").get(0);
-        assertEquals("hdfs://" + NAMESERVICE, url);
+        // Create the URL
+        List<String> urls = 
ServiceURLFactory.newInstance(cluster).create("NAMENODE", serviceParams);
+        assertNotNull(urls);
+        assertFalse(urls.isEmpty());
+        return urls.get(0);
     }
 
 
@@ -271,7 +298,7 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, 
mappingConfiguration);
-        String url = builder.create("WEBHCAT").get(0);
+        String url = builder.create("WEBHCAT", null).get(0);
         assertEquals("http://"; + HOSTNAME + ":" + PORT + "/templeton", url);
     }
 
@@ -298,7 +325,7 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, 
mappingConfiguration);
-        String url = builder.create("OOZIE").get(0);
+        String url = builder.create("OOZIE", null).get(0);
         assertEquals(URL, url);
     }
 
@@ -326,27 +353,18 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, 
mappingConfiguration);
-        List<String> urls = builder.create("WEBHBASE");
+        List<String> urls = builder.create("WEBHBASE", null);
         validateServiceURLs(urls, HOSTNAMES, "http", "60080", null);
     }
 
     @Test
-    public void testWebHdfsURLFromInternalMapping() throws Exception {
-        testWebHdfsURL(null);
-    }
-
-    @Test
-    public void testWebHdfsURLFromExternalMapping() throws Exception {
-        testWebHdfsURL(TEST_MAPPING_CONFIG);
-    }
-
-    private void testWebHdfsURL(Object mappingConfiguration) throws Exception {
+    public void testWebHdfsURL() throws Exception {
         final String ADDRESS = "host3:1357";
-        assertEquals("http://"; + ADDRESS + "/webhdfs", 
getTestWebHdfsURL(ADDRESS, mappingConfiguration));
+        assertEquals("http://"; + ADDRESS + "/webhdfs", 
getTestWebHdfsURL(ADDRESS));
     }
 
 
-    private String getTestWebHdfsURL(String address, Object 
mappingConfiguration) throws Exception {
+    private String getTestWebHdfsURL(String address) throws Exception {
         AmbariCluster.ServiceConfiguration hdfsSC = 
EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
         Map<String, String> hdfsProps = new HashMap<>();
         hdfsProps.put("dfs.namenode.http-address", address);
@@ -358,14 +376,14 @@ public class AmbariDynamicServiceURLCreatorTest {
         EasyMock.replay(cluster);
 
         // Create the URL
-        List<String> urls = 
ServiceURLFactory.newInstance(cluster).create("WEBHDFS");
+        List<String> urls = 
ServiceURLFactory.newInstance(cluster).create("WEBHDFS", null);
         assertNotNull(urls);
         assertFalse(urls.isEmpty());
         return urls.get(0);
     }
 
     @Test
-    public void testWebHdfsURLHA() throws Exception {
+    public void testWebHdfsURLHASingleNameService() throws Exception {
         final String NAMESERVICES   = "myNameServicesCluster";
         final String HTTP_ADDRESS_1 = "host1:50070";
         final String HTTP_ADDRESS_2 = "host2:50077";
@@ -390,7 +408,262 @@ public class AmbariDynamicServiceURLCreatorTest {
         EasyMock.replay(cluster);
 
         // Create the URL
-        List<String> webhdfsURLs = 
ServiceURLFactory.newInstance(cluster).create("WEBHDFS");
+        List<String> webhdfsURLs = 
ServiceURLFactory.newInstance(cluster).create("WEBHDFS", null);
+        assertEquals(2, webhdfsURLs.size());
+        assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_1));
+        assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_2));
+    }
+
+
+    /**
+     * Test federated NameNode scenario, which chooses the "first" nameservice 
because there is no information from
+     * which one can be selected from among the set.
+     */
+    @Test
+    public void testWebHdfsURLHAMultipleNameServicesNoDefaultFS() throws 
Exception {
+        final String NS1 = "myns1";
+        final String NS2 = "myns2";
+        final String NAMESERVICES   = NS1 + "," + NS2;
+        final String HTTP_ADDRESS_11 = "host11:50070";
+        final String HTTP_ADDRESS_12 = "host12:50077";
+        final String HTTP_ADDRESS_21 = "host21:50070";
+        final String HTTP_ADDRESS_22 = "host22:50077";
+
+        final String EXPECTED_ADDR_1 = "http://"; + HTTP_ADDRESS_11 + 
"/webhdfs";
+        final String EXPECTED_ADDR_2 = "http://"; + HTTP_ADDRESS_12 + 
"/webhdfs";
+
+        AmbariComponent namenode = 
EasyMock.createNiceMock(AmbariComponent.class);
+        
EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes();
+        EasyMock.replay(namenode);
+
+        AmbariCluster.ServiceConfiguration hdfsSC = 
EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+        Map<String, String> hdfsProps = new HashMap<>();
+        hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn1", 
HTTP_ADDRESS_11);
+        hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn2", 
HTTP_ADDRESS_12);
+        hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn1", 
HTTP_ADDRESS_21);
+        hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn2", 
HTTP_ADDRESS_22);
+        
EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes();
+        EasyMock.replay(hdfsSC);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        
EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
+        EasyMock.expect(cluster.getServiceConfiguration("HDFS", 
"hdfs-site")).andReturn(hdfsSC).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Create the URL
+        List<String> webhdfsURLs = 
ServiceURLFactory.newInstance(cluster).create("WEBHDFS", null);
+        assertEquals(2, webhdfsURLs.size());
+        assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_1));
+        assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_2));
+    }
+
+
+    /**
+     * Test federated NameNode scenario, relying on the core-site property for 
identifying the default nameservice.
+     */
+    @Test
+    public void testWebHdfsURLFederatedNNWithDefaultFS() throws Exception {
+        final String NS1 = "myns1";
+        final String NS2 = "myns2";
+        final String NAMESERVICES   = NS1 + "," + NS2;
+        final String HTTP_ADDRESS_11 = "host11:50070";
+        final String HTTP_ADDRESS_12 = "host12:50077";
+        final String HTTP_ADDRESS_21 = "host21:50070";
+        final String HTTP_ADDRESS_22 = "host22:50077";
+
+        final String EXPECTED_ADDR_1 = "http://"; + HTTP_ADDRESS_21 + 
"/webhdfs";
+        final String EXPECTED_ADDR_2 = "http://"; + HTTP_ADDRESS_22 + 
"/webhdfs";
+
+        AmbariComponent namenode = 
EasyMock.createNiceMock(AmbariComponent.class);
+        
EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes();
+        EasyMock.replay(namenode);
+
+        AmbariCluster.ServiceConfiguration hdfsSC = 
EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+        Map<String, String> hdfsProps = new HashMap<>();
+        hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn1", 
HTTP_ADDRESS_11);
+        hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn2", 
HTTP_ADDRESS_12);
+        hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn1", 
HTTP_ADDRESS_21);
+        hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn2", 
HTTP_ADDRESS_22);
+        
EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes();
+        EasyMock.replay(hdfsSC);
+
+        AmbariCluster.ServiceConfiguration coreSC = 
EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+        Map<String, String> coreProps = new HashMap<>();
+        coreProps.put("fs.defaultFS", NS2);
+        
EasyMock.expect(coreSC.getProperties()).andReturn(coreProps).anyTimes();
+        EasyMock.replay(coreSC);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        
EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
+        EasyMock.expect(cluster.getServiceConfiguration("HDFS", 
"hdfs-site")).andReturn(hdfsSC).anyTimes();
+        EasyMock.expect(cluster.getServiceConfiguration("HDFS", 
"core-site")).andReturn(coreSC).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Create the URL
+        List<String> webhdfsURLs = 
ServiceURLFactory.newInstance(cluster).create("WEBHDFS", null);
+        assertEquals(2, webhdfsURLs.size());
+        assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_1));
+        assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_2));
+    }
+
+
+    /**
+     * Recent version of HDFS config include properties for mapping NN nodes 
to nameservices (e.g., dfs.ha.namenode.ns1).
+     * This test verifies that discovery works correctly in those cases, when 
no nameservice is explicitly declared in
+     * a descriptor.
+     */
+    @Test
+    public void testWebHdfsURLFederatedNNWithDefaultFSAndHaNodes() throws 
Exception {
+        final String NS1 = "myns1";
+        final String NS2 = "myns2";
+        final String NAMESERVICES   = NS1 + "," + NS2;
+        final String HTTP_ADDRESS_11 = "host11:50070";
+        final String HTTP_ADDRESS_12 = "host12:50077";
+        final String HTTP_ADDRESS_21 = "host21:50070";
+        final String HTTP_ADDRESS_22 = "host22:50077";
+
+        final String EXPECTED_ADDR_1 = "http://"; + HTTP_ADDRESS_21 + 
"/webhdfs";
+        final String EXPECTED_ADDR_2 = "http://"; + HTTP_ADDRESS_22 + 
"/webhdfs";
+
+        AmbariComponent namenode = 
EasyMock.createNiceMock(AmbariComponent.class);
+        
EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes();
+        EasyMock.replay(namenode);
+
+        AmbariCluster.ServiceConfiguration hdfsSC = 
EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+        Map<String, String> hdfsProps = new HashMap<>();
+        hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn11", 
HTTP_ADDRESS_11);
+        hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn12", 
HTTP_ADDRESS_12);
+        hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn21", 
HTTP_ADDRESS_21);
+        hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn22", 
HTTP_ADDRESS_22);
+        hdfsProps.put("dfs.ha.namenodes." + NS1, "nn11,nn12");
+        hdfsProps.put("dfs.ha.namenodes." + NS2, "nn21,nn22");
+        
EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes();
+        EasyMock.replay(hdfsSC);
+
+        AmbariCluster.ServiceConfiguration coreSC = 
EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+        Map<String, String> coreProps = new HashMap<>();
+        coreProps.put("fs.defaultFS", NS2);
+        
EasyMock.expect(coreSC.getProperties()).andReturn(coreProps).anyTimes();
+        EasyMock.replay(coreSC);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        
EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
+        EasyMock.expect(cluster.getServiceConfiguration("HDFS", 
"hdfs-site")).andReturn(hdfsSC).anyTimes();
+        EasyMock.expect(cluster.getServiceConfiguration("HDFS", 
"core-site")).andReturn(coreSC).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Create the URL
+        List<String> webhdfsURLs = 
ServiceURLFactory.newInstance(cluster).create("WEBHDFS", null);
+        assertEquals(2, webhdfsURLs.size());
+        assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_1));
+        assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_2));
+    }
+
+
+    /**
+     * Recent version of HDFS config include properties for mapping NN nodes 
to nameservices (e.g., dfs.ha.namenode.ns1).
+     * This test verifies that discovery works correctly in those cases, when 
a nameservice is declared in descriptor.
+     */
+    @Test
+    public void testWebHdfsURLFederatedNNDeclaredNS() throws Exception {
+        final String NS1 = "myns1";
+        final String NS2 = "myns2";
+        final String NAMESERVICES   = NS1 + "," + NS2;
+        final String HTTP_ADDRESS_11 = "host11:50070";
+        final String HTTP_ADDRESS_12 = "host12:50077";
+        final String HTTP_ADDRESS_21 = "host21:50070";
+        final String HTTP_ADDRESS_22 = "host22:50077";
+
+        final String EXPECTED_ADDR_1 = "http://"; + HTTP_ADDRESS_11 + 
"/webhdfs";
+        final String EXPECTED_ADDR_2 = "http://"; + HTTP_ADDRESS_12 + 
"/webhdfs";
+
+        AmbariComponent namenode = 
EasyMock.createNiceMock(AmbariComponent.class);
+        
EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes();
+        EasyMock.replay(namenode);
+
+        AmbariCluster.ServiceConfiguration hdfsSC = 
EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+        Map<String, String> hdfsProps = new HashMap<>();
+        hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn11", 
HTTP_ADDRESS_11);
+        hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn12", 
HTTP_ADDRESS_12);
+        hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn21", 
HTTP_ADDRESS_21);
+        hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn22", 
HTTP_ADDRESS_22);
+        hdfsProps.put("dfs.ha.namenodes." + NS1, "nn11,nn12");
+        hdfsProps.put("dfs.ha.namenodes." + NS2, "nn21,nn22");
+        hdfsProps.put("dfs.nameservices", NAMESERVICES);
+        
EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes();
+        EasyMock.replay(hdfsSC);
+
+        AmbariCluster.ServiceConfiguration coreSC = 
EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+        Map<String, String> coreProps = new HashMap<>();
+        coreProps.put("fs.defaultFS", NS2);
+        
EasyMock.expect(coreSC.getProperties()).andReturn(coreProps).anyTimes();
+        EasyMock.replay(coreSC);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        
EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
+        EasyMock.expect(cluster.getServiceConfiguration("HDFS", 
"hdfs-site")).andReturn(hdfsSC).anyTimes();
+        EasyMock.expect(cluster.getServiceConfiguration("HDFS", 
"core-site")).andReturn(coreSC).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Create the URL
+        Map<String, String> params = new HashMap<>();
+        params.put("discovery-nameservice", NS1); // Declare the non-default 
nameservice
+        List<String> webhdfsURLs = 
ServiceURLFactory.newInstance(cluster).create("WEBHDFS", params);
+
+        assertEquals(2, webhdfsURLs.size());
+        assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_1));
+        assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_2));
+    }
+
+
+    /**
+     * Previous version of HDFS config DO NOT include properties for mapping 
NN nodes to nameservices.
+     * This test verifies that discovery works correctly in those cases, when 
a nameservice is declared in descriptor.
+     */
+    @Test
+    public void testWebHdfsURLFederatedNNDeclaredNSWithoutHaNodes() throws 
Exception {
+        final String NS1 = "myns1";
+        final String NS2 = "myns2";
+        final String NAMESERVICES   = NS1 + "," + NS2;
+        final String HTTP_ADDRESS_11 = "host11:50070";
+        final String HTTP_ADDRESS_12 = "host12:50077";
+        final String HTTP_ADDRESS_21 = "host21:50070";
+        final String HTTP_ADDRESS_22 = "host22:50077";
+
+        final String EXPECTED_ADDR_1 = "http://"; + HTTP_ADDRESS_11 + 
"/webhdfs";
+        final String EXPECTED_ADDR_2 = "http://"; + HTTP_ADDRESS_12 + 
"/webhdfs";
+
+        AmbariComponent namenode = 
EasyMock.createNiceMock(AmbariComponent.class);
+        
EasyMock.expect(namenode.getConfigProperty("dfs.nameservices")).andReturn(NAMESERVICES).anyTimes();
+        EasyMock.replay(namenode);
+
+        AmbariCluster.ServiceConfiguration hdfsSC = 
EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+        Map<String, String> hdfsProps = new HashMap<>();
+        hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn1", 
HTTP_ADDRESS_11);
+        hdfsProps.put("dfs.namenode.http-address." + NS1 + ".nn2", 
HTTP_ADDRESS_12);
+        hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn1", 
HTTP_ADDRESS_21);
+        hdfsProps.put("dfs.namenode.http-address." + NS2 + ".nn2", 
HTTP_ADDRESS_22);
+        hdfsProps.put("dfs.nameservices", NAMESERVICES);
+        
EasyMock.expect(hdfsSC.getProperties()).andReturn(hdfsProps).anyTimes();
+        EasyMock.replay(hdfsSC);
+
+        AmbariCluster.ServiceConfiguration coreSC = 
EasyMock.createNiceMock(AmbariCluster.ServiceConfiguration.class);
+        Map<String, String> coreProps = new HashMap<>();
+        coreProps.put("fs.defaultFS", NS2);
+        
EasyMock.expect(coreSC.getProperties()).andReturn(coreProps).anyTimes();
+        EasyMock.replay(coreSC);
+
+        AmbariCluster cluster = EasyMock.createNiceMock(AmbariCluster.class);
+        
EasyMock.expect(cluster.getComponent("NAMENODE")).andReturn(namenode).anyTimes();
+        EasyMock.expect(cluster.getServiceConfiguration("HDFS", 
"hdfs-site")).andReturn(hdfsSC).anyTimes();
+        EasyMock.expect(cluster.getServiceConfiguration("HDFS", 
"core-site")).andReturn(coreSC).anyTimes();
+        EasyMock.replay(cluster);
+
+        // Create the URL
+        Map<String, String> params = new HashMap<>();
+        params.put("discovery-nameservice", NS1); // Declare the non-default 
nameservice
+        List<String> webhdfsURLs = 
ServiceURLFactory.newInstance(cluster).create("WEBHDFS", params);
+
         assertEquals(2, webhdfsURLs.size());
         assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_1));
         assertTrue(webhdfsURLs.contains(EXPECTED_ADDR_2));
@@ -411,7 +684,7 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
-        List<String> urls = builder.create("ATLAS-API");
+        List<String> urls = builder.create("ATLAS-API", null);
         assertEquals(1, urls.size());
         assertEquals(ATLAS_REST_ADDRESS, urls.get(0));
     }
@@ -438,7 +711,7 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
-        List<String> urls = builder.create("ATLAS");
+        List<String> urls = builder.create("ATLAS", null);
         validateServiceURLs(urls, HOSTNAMES, "http", HTTP_PORT, null);
 
         EasyMock.reset(atlasServer);
@@ -449,7 +722,7 @@ public class AmbariDynamicServiceURLCreatorTest {
         EasyMock.replay(atlasServer);
 
         // Run the test
-        urls = builder.create("ATLAS");
+        urls = builder.create("ATLAS", null);
         validateServiceURLs(urls, HOSTNAMES, "https", HTTPS_PORT, null);
     }
 
@@ -476,7 +749,7 @@ public class AmbariDynamicServiceURLCreatorTest {
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
 
         // Run the test
-        validateServiceURLs(builder.create("ZEPPELIN"), HOSTNAMES, "http", 
HTTP_PORT, null);
+        validateServiceURLs(builder.create("ZEPPELIN", null), HOSTNAMES, 
"http", HTTP_PORT, null);
 
         EasyMock.reset(zeppelinMaster);
         
EasyMock.expect(zeppelinMaster.getHostNames()).andReturn(atlastServerHosts).anyTimes();
@@ -486,7 +759,7 @@ public class AmbariDynamicServiceURLCreatorTest {
         EasyMock.replay(zeppelinMaster);
 
         // Run the test
-        validateServiceURLs(builder.create("ZEPPELIN"), HOSTNAMES, "https", 
HTTPS_PORT, null);
+        validateServiceURLs(builder.create("ZEPPELIN", null), HOSTNAMES, 
"https", HTTPS_PORT, null);
     }
 
 
@@ -512,7 +785,7 @@ public class AmbariDynamicServiceURLCreatorTest {
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
 
         // Run the test
-        validateServiceURLs(builder.create("ZEPPELINUI"), HOSTNAMES, "http", 
HTTP_PORT, null);
+        validateServiceURLs(builder.create("ZEPPELINUI", null), HOSTNAMES, 
"http", HTTP_PORT, null);
 
         EasyMock.reset(zeppelinMaster);
         
EasyMock.expect(zeppelinMaster.getHostNames()).andReturn(atlastServerHosts).anyTimes();
@@ -522,7 +795,7 @@ public class AmbariDynamicServiceURLCreatorTest {
         EasyMock.replay(zeppelinMaster);
 
         // Run the test
-        validateServiceURLs(builder.create("ZEPPELINUI"), HOSTNAMES, "https", 
HTTPS_PORT, null);
+        validateServiceURLs(builder.create("ZEPPELINUI", null), HOSTNAMES, 
"https", HTTPS_PORT, null);
     }
 
 
@@ -548,7 +821,7 @@ public class AmbariDynamicServiceURLCreatorTest {
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
 
         // Run the test
-        validateServiceURLs(builder.create("ZEPPELINWS"), HOSTNAMES, "ws", 
HTTP_PORT, null);
+        validateServiceURLs(builder.create("ZEPPELINWS", null), HOSTNAMES, 
"ws", HTTP_PORT, null);
 
         EasyMock.reset(zeppelinMaster);
         
EasyMock.expect(zeppelinMaster.getHostNames()).andReturn(atlastServerHosts).anyTimes();
@@ -558,7 +831,7 @@ public class AmbariDynamicServiceURLCreatorTest {
         EasyMock.replay(zeppelinMaster);
 
         // Run the test
-        validateServiceURLs(builder.create("ZEPPELINWS"), HOSTNAMES, "wss", 
HTTPS_PORT, null);
+        validateServiceURLs(builder.create("ZEPPELINWS", null), HOSTNAMES, 
"wss", HTTPS_PORT, null);
     }
 
 
@@ -580,7 +853,7 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
-        List<String> urls = builder.create("DRUID-COORDINATOR");
+        List<String> urls = builder.create("DRUID-COORDINATOR", null);
         validateServiceURLs(urls, HOSTNAMES, "http", PORT, null);
     }
 
@@ -603,7 +876,7 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
-        List<String> urls = builder.create("DRUID-BROKER");
+        List<String> urls = builder.create("DRUID-BROKER", null);
         validateServiceURLs(urls, HOSTNAMES, "http", PORT, null);
     }
 
@@ -626,7 +899,7 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
-        List<String> urls = builder.create("DRUID-ROUTER");
+        List<String> urls = builder.create("DRUID-ROUTER", null);
         validateServiceURLs(urls, HOSTNAMES, "http", PORT, null);
     }
 
@@ -649,7 +922,7 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
-        List<String> urls = builder.create("DRUID-OVERLORD");
+        List<String> urls = builder.create("DRUID-OVERLORD", null);
         validateServiceURLs(urls, HOSTNAMES, "http", PORT, null);
     }
 
@@ -672,7 +945,7 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
-        List<String> urls = builder.create("SUPERSET");
+        List<String> urls = builder.create("SUPERSET", null);
         validateServiceURLs(urls, HOSTNAMES, "http", PORT, null);
     }
 
@@ -686,12 +959,12 @@ public class AmbariDynamicServiceURLCreatorTest {
 
         // Run the test
         AmbariDynamicServiceURLCreator builder = newURLCreator(cluster, null);
-        List<String> urls = builder.create("DRUID-BROKER");
+        List<String> urls = builder.create("DRUID-BROKER", null);
         assertNotNull(urls);
         assertEquals(1, urls.size());
         assertEquals("http://{HOST}:{PORT}";, urls.get(0));
 
-        urls = builder.create("HIVE");
+        urls = builder.create("HIVE", null);
         assertNotNull(urls);
         assertEquals(1, urls.size());
         assertEquals("http://{HOST}:{PORT}/{PATH}";, urls.get(0));

http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
----------------------------------------------------------------------
diff --git 
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
 
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
index 55411d8..01e6d43 100644
--- 
a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
+++ 
b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java
@@ -76,6 +76,8 @@ public class SimpleDescriptorHandler {
 
     private static final SimpleDescriptorMessages log = 
MessagesFactory.get(SimpleDescriptorMessages.class);
 
+    private static final String DISCOVERY_PARAM_PREFIX = "discovery-";
+
     private static Map<String, ServiceDiscovery> discoveryInstances = new 
HashMap<>();
 
     public static Map<String, File> handle(File desc) throws IOException {
@@ -112,7 +114,7 @@ public class SimpleDescriptorHandler {
 
                 List<String> descServiceURLs = descService.getURLs();
                 if (descServiceURLs == null || descServiceURLs.isEmpty()) {
-                    descServiceURLs = cluster.getServiceURLs(serviceName);
+                    descServiceURLs = cluster.getServiceURLs(serviceName, 
descService.getParams());
                 }
 
                 // Validate the discovered service URLs
@@ -440,10 +442,12 @@ public class SimpleDescriptorHandler {
                 Map<String, String> svcParams = serviceParams.get(serviceName);
                 if (svcParams != null) {
                     for (String paramName : svcParams.keySet()) {
-                        sw.write("        <param>\n");
-                        sw.write("            <name>" + paramName + 
"</name>\n");
-                        sw.write("            <value>" + 
svcParams.get(paramName) + "</value>\n");
-                        sw.write("        </param>\n");
+                        if 
(!(paramName.toLowerCase()).startsWith(DISCOVERY_PARAM_PREFIX)) {
+                            sw.write("        <param>\n");
+                            sw.write("            <name>" + paramName + 
"</name>\n");
+                            sw.write("            <value>" + 
svcParams.get(paramName) + "</value>\n");
+                            sw.write("        </param>\n");
+                        }
                     }
                 }
 

http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/DummyServiceDiscovery.java
----------------------------------------------------------------------
diff --git 
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/DummyServiceDiscovery.java
 
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/DummyServiceDiscovery.java
index 21883d9..4ac88d3 100644
--- 
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/DummyServiceDiscovery.java
+++ 
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/DummyServiceDiscovery.java
@@ -44,6 +44,11 @@ public class DummyServiceDiscovery implements 
ServiceDiscovery {
         }
 
         @Override
+        public List<String> getServiceURLs(String serviceName, Map<String, 
String> serviceParams) {
+            return getServiceURLs(serviceName);
+        }
+
+        @Override
         public ZooKeeperConfig getZooKeeperConfiguration(String serviceName) {
             return null;
         }

http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/PropertiesFileServiceDiscovery.java
----------------------------------------------------------------------
diff --git 
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/PropertiesFileServiceDiscovery.java
 
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/PropertiesFileServiceDiscovery.java
index 45307f1..1d89733 100644
--- 
a/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/PropertiesFileServiceDiscovery.java
+++ 
b/gateway-server/src/test/java/org/apache/knox/gateway/topology/discovery/test/extension/PropertiesFileServiceDiscovery.java
@@ -101,6 +101,11 @@ class PropertiesFileServiceDiscovery implements 
ServiceDiscovery {
 
         @Override
         public List<String> getServiceURLs(String serviceName) {
+            return getServiceURLs(serviceName, null);
+        }
+
+        @Override
+        public List<String> getServiceURLs(String serviceName, Map<String, 
String> serviceParams) {
             return serviceURLS.get(serviceName);
         }
 

http://git-wip-us.apache.org/repos/asf/knox/blob/165ea0fd/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java
----------------------------------------------------------------------
diff --git 
a/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java
 
b/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java
index b3366a3..01b2178 100644
--- 
a/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java
+++ 
b/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java
@@ -67,12 +67,22 @@ public interface ServiceDiscovery {
 
         /**
          * @param serviceName The name of the service
+         *
          * @return The URLs for the specified service in this cluster.
          */
         List<String> getServiceURLs(String serviceName);
 
         /**
+         * @param serviceName   The name of the service.
+         * @param serviceParams A map of parameters and their corresponding 
values for the specified service.
+         *
+         * @return The URLs for the specified service in this cluster.
+         */
+        List<String> getServiceURLs(String serviceName, Map<String, String> 
serviceParams);
+
+        /**
          * @param serviceName The name of the service
+         *
          * @return The HA configuration properties for the specified service 
in this cluster.
          */
         ZooKeeperConfig getZooKeeperConfiguration(String serviceName);

Reply via email to