Repository: knox Updated Branches: refs/heads/master 165ea0fde -> d402f6d23
http://git-wip-us.apache.org/repos/asf/knox/blob/d402f6d2/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptor.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptor.java b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptor.java index 7d25286..a27adc8 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptor.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptor.java @@ -43,6 +43,8 @@ public interface SimpleDescriptor { interface Service { String getName(); + String getVersion(); + Map<String, String> getParams(); List<String> getURLs(); http://git-wip-us.apache.org/repos/asf/knox/blob/d402f6d2/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java index 01e6d43..d24a6cf 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java @@ -103,6 +103,7 @@ public class SimpleDescriptorHandler { public static Map<String, File> handle(SimpleDescriptor desc, File srcDirectory, File destDirectory, Service...gatewayServices) { List<String> validServiceNames = new ArrayList<>(); + Map<String, String> serviceVersions = new HashMap<>(); Map<String, Map<String, String>> serviceParams = new HashMap<>(); Map<String, List<String>> serviceURLs = new HashMap<>(); @@ -112,6 +113,11 @@ public class SimpleDescriptorHandler { for (SimpleDescriptor.Service descService : desc.getServices()) { String serviceName = descService.getName(); + String serviceVer = descService.getVersion(); + if (serviceVer != null) { + serviceVersions.put(serviceName, serviceVer); + } + List<String> descServiceURLs = descService.getURLs(); if (descServiceURLs == null || descServiceURLs.isEmpty()) { descServiceURLs = cluster.getServiceURLs(serviceName, descService.getParams()); @@ -160,7 +166,7 @@ public class SimpleDescriptorHandler { } // Generate the topology file - return generateTopology(desc, srcDirectory, destDirectory, cluster, validServiceNames, serviceURLs, serviceParams); + return generateTopology(desc, srcDirectory, destDirectory, cluster, validServiceNames, serviceVersions, serviceURLs, serviceParams); } @@ -316,6 +322,7 @@ public class SimpleDescriptorHandler { * @param destDirectory The destination directory for the generated topology file. * @param cluster The discovery details for the referenced cluster. * @param validServiceNames The validated service names. + * @param serviceVersions The versions of the services; optional attribute. * @param serviceURLs The URLs associated with the valid service names. * @param serviceParams The params associated with the valid service names. * @@ -326,6 +333,7 @@ public class SimpleDescriptorHandler { final File destDirectory, final ServiceDiscovery.Cluster cluster, final List<String> validServiceNames, + final Map<String, String> serviceVersions, final Map<String, List<String>> serviceURLs, final Map<String, Map<String, String>> serviceParams) { Map<String, File> result = new HashMap<>(); @@ -413,6 +421,10 @@ public class SimpleDescriptorHandler { sw.write(" <service>\n"); sw.write(" <role>" + serviceName + "</role>\n"); + if (serviceVersions.containsKey(serviceName)) { + sw.write(" <version>" + serviceVersions.get(serviceName) + "</version>\n"); + } + // If the service is configured for ZooKeeper-based HA ServiceDiscovery.Cluster.ZooKeeperConfig zkConf = haServiceParams.get(serviceName); if (zkConf != null && zkConf.isEnabled() && zkConf.getEnsemble() != null) { http://git-wip-us.apache.org/repos/asf/knox/blob/d402f6d2/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorImpl.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorImpl.java b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorImpl.java index f3288fd..3ad2f88 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorImpl.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorImpl.java @@ -112,6 +112,9 @@ class SimpleDescriptorImpl implements SimpleDescriptor { @JsonProperty("name") private String name; + @JsonProperty("version") + private String version; + @JsonProperty("params") private Map<String, String> params; @@ -124,6 +127,11 @@ class SimpleDescriptorImpl implements SimpleDescriptor { } @Override + public String getVersion() { + return version; + } + + @Override public Map<String, String> getParams() { return params; } http://git-wip-us.apache.org/repos/asf/knox/blob/d402f6d2/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorFactoryTest.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorFactoryTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorFactoryTest.java index 2622f13..0fdabee 100644 --- a/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorFactoryTest.java +++ b/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorFactoryTest.java @@ -45,6 +45,17 @@ public class SimpleDescriptorFactoryTest { } @Test + public void testParseJSONSimpleDescriptorWithServiceVersions() throws Exception { + testParseSimpleDescriptorWithServiceVersions(FileType.JSON); + } + + @Test + public void testParseYAMLSimpleDescriptorWithServiceVersions() throws Exception { + testParseSimpleDescriptorWithServiceVersions(FileType.YML); + testParseSimpleDescriptorWithServiceVersions(FileType.YAML); + } + + @Test public void testParseJSONSimpleDescriptorWithServiceParams() throws Exception { testParseSimpleDescriptorWithServiceParams(FileType.JSON); } @@ -90,7 +101,9 @@ public class SimpleDescriptorFactoryTest { services.put("NODEMANAGER", null); services.put("JOBTRACKER", null); services.put("RESOURCEMANAGER", null); - services.put("HIVE", Arrays.asList("http://c6401.ambari.apache.org";, "http://c6402.ambari.apache.org";, "http://c6403.ambari.apache.org";)); + services.put("HIVE", Arrays.asList("http://c6401.ambari.apache.org";, + "http://c6402.ambari.apache.org";, + "http://c6403.ambari.apache.org";)); services.put("AMBARIUI", Collections.singletonList("http://c6401.ambari.apache.org:8080";)); String fileName = "test-topology." + getFileExtensionForType(type); @@ -119,6 +132,55 @@ public class SimpleDescriptorFactoryTest { } } + private void testParseSimpleDescriptorWithServiceVersions(FileType type) throws Exception { + final String discoveryType = "AMBARI"; + final String discoveryAddress = "http://c6401.ambari.apache.org:8080";; + final String discoveryUser = "joeblow"; + final String providerConfig = "ambari-cluster-policy.xml"; + final String clusterName = "myCluster"; + + final Map<String, List<String>> services = new HashMap<>(); + services.put("NODEMANAGER", null); + services.put("JOBTRACKER", null); + services.put("RESOURCEMANAGER", null); + services.put("WEBHDFS", null); + services.put("HIVE", Arrays.asList("http://c6401.ambari.apache.org";, + "http://c6402.ambari.apache.org";, + "http://c6403.ambari.apache.org";)); + services.put("AMBARIUI", Collections.singletonList("http://c6401.ambari.apache.org:8080";)); + + Map<String, String> serviceVersions = new HashMap<>(); + serviceVersions.put("HIVE", "0.13.0"); + serviceVersions.put("WEBHDFS", "2.4.0"); + + String fileName = "test-topology." + getFileExtensionForType(type); + File testFile = null; + try { + testFile = writeDescriptorFile(type, + fileName, + discoveryType, + discoveryAddress, + discoveryUser, + providerConfig, + clusterName, + services, + serviceVersions); + SimpleDescriptor sd = SimpleDescriptorFactory.parse(testFile.getAbsolutePath()); + validateSimpleDescriptor(sd, discoveryType, discoveryAddress, providerConfig, clusterName, services, serviceVersions); + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (testFile != null) { + try { + testFile.delete(); + } catch (Exception e) { + // Ignore + } + } + } + } + + private void testParseSimpleDescriptorWithServiceParams(FileType type) throws Exception { final String discoveryType = "AMBARI"; @@ -165,9 +227,10 @@ public class SimpleDescriptorFactoryTest { providerConfig, clusterName, services, + null, serviceParams); SimpleDescriptor sd = SimpleDescriptorFactory.parse(testFile.getAbsolutePath()); - validateSimpleDescriptor(sd, discoveryType, discoveryAddress, providerConfig, clusterName, services, serviceParams); + validateSimpleDescriptor(sd, discoveryType, discoveryAddress, providerConfig, clusterName, services, null, serviceParams); } finally { if (testFile != null) { try { @@ -216,6 +279,7 @@ public class SimpleDescriptorFactoryTest { clusterName, null, null, + null, apps, appParams); SimpleDescriptor sd = SimpleDescriptorFactory.parse(testFile.getAbsolutePath()); @@ -226,6 +290,7 @@ public class SimpleDescriptorFactoryTest { clusterName, null, null, + null, apps, appParams); } finally { @@ -302,6 +367,7 @@ public class SimpleDescriptorFactoryTest { providerConfig, clusterName, services, + null, serviceParams, apps, appParams); @@ -312,6 +378,7 @@ public class SimpleDescriptorFactoryTest { providerConfig, clusterName, services, + null, serviceParams, apps, appParams); @@ -342,14 +409,14 @@ public class SimpleDescriptorFactoryTest { return extension; } - private File writeDescriptorFile(FileType type, - String path, - String discoveryType, - String discoveryAddress, - String discoveryUser, - String providerConfig, - String clusterName, - Map<String, List<String>> services) throws Exception { + private File writeDescriptorFile(FileType type, + String path, + String discoveryType, + String discoveryAddress, + String discoveryUser, + String providerConfig, + String clusterName, + Map<String, List<String>> services) throws Exception { return writeDescriptorFile(type, path, discoveryType, @@ -361,7 +428,28 @@ public class SimpleDescriptorFactoryTest { null); } - private File writeDescriptorFile(FileType type, + private File writeDescriptorFile(FileType type, + String path, + String discoveryType, + String discoveryAddress, + String discoveryUser, + String providerConfig, + String clusterName, + Map<String, List<String>> services, + Map<String, String> serviceVersions) throws Exception { + return writeDescriptorFile(type, + path, + discoveryType, + discoveryAddress, + discoveryUser, + providerConfig, + clusterName, + services, + serviceVersions, + null); + } + + private File writeDescriptorFile(FileType type, String path, String discoveryType, String discoveryAddress, @@ -369,6 +457,7 @@ public class SimpleDescriptorFactoryTest { String providerConfig, String clusterName, Map<String, List<String>> services, + Map<String, String> serviceVersions, Map<String, Map<String, String>> serviceParams) throws Exception { return writeDescriptorFile(type, path, @@ -378,13 +467,14 @@ public class SimpleDescriptorFactoryTest { providerConfig, clusterName, services, + serviceVersions, serviceParams, null, null); } - private File writeDescriptorFile(FileType type, + private File writeDescriptorFile(FileType type, String path, String discoveryType, String discoveryAddress, @@ -392,6 +482,7 @@ public class SimpleDescriptorFactoryTest { String providerConfig, String clusterName, Map<String, List<String>> services, + Map<String, String> serviceVersions, Map<String, Map<String, String>> serviceParams, Map<String, List<String>> apps, Map<String, Map<String, String>> appParams) throws Exception { @@ -405,6 +496,7 @@ public class SimpleDescriptorFactoryTest { providerConfig, clusterName, services, + serviceVersions, serviceParams, apps, appParams); @@ -418,6 +510,7 @@ public class SimpleDescriptorFactoryTest { providerConfig, clusterName, services, + serviceVersions, serviceParams, apps, appParams); @@ -427,15 +520,16 @@ public class SimpleDescriptorFactoryTest { } - private File writeJSON(String path, - String discoveryType, - String discoveryAddress, - String discoveryUser, - String providerConfig, - String clusterName, - Map<String, List<String>> services, + private File writeJSON(String path, + String discoveryType, + String discoveryAddress, + String discoveryUser, + String providerConfig, + String clusterName, + Map<String, List<String>> services, + Map<String, String> serviceVersions, Map<String, Map<String, String>> serviceParams, - Map<String, List<String>> apps, + Map<String, List<String>> apps, Map<String, Map<String, String>> appParams) throws Exception { File f = new File(path); @@ -449,13 +543,13 @@ public class SimpleDescriptorFactoryTest { if (services != null && !services.isEmpty()) { fw.write(",\n\"services\":[\n"); - writeServiceOrApplicationJSON(fw, services, serviceParams); + writeServiceOrApplicationJSON(fw, services, serviceParams, serviceVersions); fw.write("]\n"); } if (apps != null && !apps.isEmpty()) { fw.write(",\n\"applications\":[\n"); - writeServiceOrApplicationJSON(fw, apps, appParams); + writeServiceOrApplicationJSON(fw, apps, appParams, null); fw.write("]\n"); } @@ -467,13 +561,21 @@ public class SimpleDescriptorFactoryTest { } private void writeServiceOrApplicationJSON(Writer fw, - Map<String, List<String>> elementURLs, - Map<String, Map<String, String>> elementParams) throws Exception { + Map<String, List<String>> elementURLs, + Map<String, Map<String, String>> elementParams, + Map<String, String> serviceVersions) throws Exception { if (elementURLs != null) { int i = 0; for (String name : elementURLs.keySet()) { fw.write("{\"name\":\"" + name + "\""); + if (serviceVersions != null) { + String ver = serviceVersions.get(name); + if (ver != null) { + fw.write(",\n\"version\":\"" + ver + "\""); + } + } + // Service params if (elementParams != null && !elementParams.isEmpty()) { Map<String, String> params = elementParams.get(name); @@ -520,6 +622,7 @@ public class SimpleDescriptorFactoryTest { String providerConfig, String clusterName, Map<String, List<String>> services, + Map<String, String> serviceVersions, Map<String, Map<String, String>> serviceParams, Map<String, List<String>> apps, Map<String, Map<String, String>> appParams) throws Exception { @@ -536,12 +639,12 @@ public class SimpleDescriptorFactoryTest { if (services != null && !services.isEmpty()) { fw.write("services:\n"); - writeServiceOrApplicationYAML(fw, services, serviceParams); + writeServiceOrApplicationYAML(fw, services, serviceParams, serviceVersions); } if (apps != null && !apps.isEmpty()) { fw.write("applications:\n"); - writeServiceOrApplicationYAML(fw, apps, appParams); + writeServiceOrApplicationYAML(fw, apps, appParams, null); } fw.flush(); @@ -552,10 +655,18 @@ public class SimpleDescriptorFactoryTest { private void writeServiceOrApplicationYAML(Writer fw, Map<String, List<String>> elementURLs, - Map<String, Map<String, String>> elementParams) throws Exception { + Map<String, Map<String, String>> elementParams, + Map<String, String> serviceVersions) throws Exception { for (String name : elementURLs.keySet()) { fw.write(" - name: " + name + "\n"); + if (serviceVersions != null) { + String ver = serviceVersions.get(name); + if (ver != null) { + fw.write(" version: " + ver + "\n"); + } + } + // Service params if (elementParams != null && !elementParams.isEmpty()) { if (elementParams.containsKey(name)) { @@ -589,12 +700,24 @@ public class SimpleDescriptorFactoryTest { } + private void validateSimpleDescriptor(SimpleDescriptor sd, + String discoveryType, + String discoveryAddress, + String providerConfig, + String clusterName, + Map<String, List<String>> expectedServices, + Map<String, String> expectedServiceVersions) { + validateSimpleDescriptor(sd, discoveryType, discoveryAddress, providerConfig, clusterName, expectedServices, expectedServiceVersions, null); + } + + private void validateSimpleDescriptor(SimpleDescriptor sd, String discoveryType, String discoveryAddress, String providerConfig, String clusterName, Map<String, List<String>> expectedServices, + Map<String, String> expectedServiceVersions, Map<String, Map<String, String>> expectedServiceParameters) { validateSimpleDescriptor(sd, discoveryType, @@ -602,6 +725,7 @@ public class SimpleDescriptorFactoryTest { providerConfig, clusterName, expectedServices, + expectedServiceVersions, expectedServiceParameters, null, null); @@ -613,6 +737,7 @@ public class SimpleDescriptorFactoryTest { String providerConfig, String clusterName, Map<String, List<String>> expectedServices, + Map<String, String> expectedServiceVersions, Map<String, Map<String, String>> expectedServiceParameters, Map<String, List<String>> expectedApps, Map<String, Map<String, String>> expectedAppParameters) { @@ -633,6 +758,17 @@ public class SimpleDescriptorFactoryTest { assertTrue(expectedServices.containsKey(actualService.getName())); assertEquals(expectedServices.get(actualService.getName()), actualService.getURLs()); + if (expectedServiceVersions != null) { + String expectedVersion = expectedServiceVersions.get(actualService.getName()); + if (expectedVersion != null) { + String actualVersion = actualService.getVersion(); + assertNotNull(actualVersion); + assertEquals("Unexpected version for " + actualService.getName(), + expectedVersion, + actualVersion); + } + } + // Validate service parameters if (expectedServiceParameters != null) { if (expectedServiceParameters.containsKey(actualService.getName())) { http://git-wip-us.apache.org/repos/asf/knox/blob/d402f6d2/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandlerTest.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandlerTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandlerTest.java index 0a64e60..6850bb6 100644 --- a/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandlerTest.java +++ b/gateway-server/src/test/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandlerTest.java @@ -187,6 +187,7 @@ public class SimpleDescriptorHandlerTest { for (String serviceName : serviceURLs.keySet()) { SimpleDescriptor.Service svc = EasyMock.createNiceMock(SimpleDescriptor.Service.class); EasyMock.expect(svc.getName()).andReturn(serviceName).anyTimes(); + EasyMock.expect(svc.getVersion()).andReturn("WEBHDFS".equals(serviceName) ? "2.4.0" : null).anyTimes(); EasyMock.expect(svc.getURLs()).andReturn(serviceURLs.get(serviceName)).anyTimes(); EasyMock.expect(svc.getParams()).andReturn(serviceParameters.get(serviceName)).anyTimes(); EasyMock.replay(svc); @@ -240,6 +241,14 @@ public class SimpleDescriptorHandlerTest { assertNotNull(roleNode); String role = roleNode.getNodeValue(); + // Validate the explicit version for the WEBHDFS service + if ("WEBHDFS".equals(role)) { + Node versionNode = (Node) xpath.compile("version/text()").evaluate(serviceNode, XPathConstants.NODE); + assertNotNull(versionNode); + String version = versionNode.getNodeValue(); + assertEquals("2.4.0", version); + } + // Validate the URLs NodeList urlNodes = (NodeList) xpath.compile("url/text()").evaluate(serviceNode, XPathConstants.NODESET); for(int urlNodeIndex = 0 ; urlNodeIndex < urlNodes.getLength(); urlNodeIndex++) {
