KNOX-1105 - Provide indication that topologies were generated from simple descriptors (Phil Zampino via lmccay)
Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/f52dc201 Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/f52dc201 Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/f52dc201 Branch: refs/heads/KNOX-998-Package_Restructuring Commit: f52dc20159ae1d8adcd6e9f9966b7bb6b42d14cf Parents: a068eb8 Author: Larry McCay <[email protected]> Authored: Wed Nov 8 19:04:40 2017 -0500 Committer: Larry McCay <[email protected]> Committed: Wed Nov 8 19:04:40 2017 -0500 ---------------------------------------------------------------------- .../builder/BeanPropertyTopologyBuilder.java | 11 +++++++++ .../simple/SimpleDescriptorHandler.java | 20 +++++++++++++++- .../xml/KnoxFormatXmlTopologyRules.java | 2 ++ .../src/main/resources/conf/topology-v1.xsd | 1 + .../simple/SimpleDescriptorHandlerTest.java | 8 +++++++ .../service/admin/TopologiesResource.java | 25 ++++++++++++++++---- .../service/admin/beans/BeanConverter.java | 2 ++ .../gateway/service/admin/beans/Topology.java | 11 +++++++++ .../hadoop/gateway/i18n/GatewaySpiMessages.java | 3 +++ .../hadoop/gateway/topology/Topology.java | 9 +++++++ 10 files changed, 87 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java index 65278a1..5058508 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java @@ -28,6 +28,7 @@ public class BeanPropertyTopologyBuilder implements TopologyBuilder { private String name; private String defaultService; + private boolean isGenerated; private List<Provider> providers; private List<Service> services; private List<Application> applications; @@ -47,6 +48,15 @@ public class BeanPropertyTopologyBuilder implements TopologyBuilder { return name; } + public BeanPropertyTopologyBuilder generated(String isGenerated) { + this.isGenerated = Boolean.valueOf(isGenerated); + return this; + } + + public boolean isGenerated() { + return isGenerated; + } + public BeanPropertyTopologyBuilder defaultService(String defaultService) { this.defaultService = defaultService; return this; @@ -87,6 +97,7 @@ public class BeanPropertyTopologyBuilder implements TopologyBuilder { Topology topology = new Topology(); topology.setName(name); topology.setDefaultServicePath(defaultService); + topology.setGenerated(isGenerated); for (Provider provider : providers) { topology.addProvider(provider); http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java index 040eb86..089925d 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java @@ -77,7 +77,14 @@ public class SimpleDescriptorHandler { DefaultServiceDiscoveryConfig sdc = new DefaultServiceDiscoveryConfig(desc.getDiscoveryAddress()); sdc.setUser(desc.getDiscoveryUser()); sdc.setPasswordAlias(desc.getDiscoveryPasswordAlias()); - ServiceDiscovery sd = ServiceDiscoveryFactory.get(desc.getDiscoveryType(), gatewayServices); + + // Use the discovery type from the descriptor. If it's unspecified, employ the default type. + String discoveryType = desc.getDiscoveryType(); + if (discoveryType == null) { + discoveryType = "AMBARI"; + } + + ServiceDiscovery sd = ServiceDiscoveryFactory.get(discoveryType, gatewayServices); ServiceDiscovery.Cluster cluster = sd.discover(sdc, desc.getClusterName()); List<String> validServiceNames = new ArrayList<>(); @@ -148,10 +155,20 @@ public class SimpleDescriptorHandler { topologyFilename = desc.getClusterName(); } topologyDescriptor = new File(destDirectory, topologyFilename + ".xml"); + fw = new BufferedWriter(new FileWriter(topologyDescriptor)); + fw.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"); + + fw.write("<!--==============================================-->\n"); + fw.write("<!-- DO NOT EDIT. This is an auto-generated file. -->\n"); + fw.write("<!--==============================================-->\n"); + fw.write("<topology>\n"); + // KNOX-1105 Indicate that this topology was auto-generated + fw.write(" <generated>true</generated>\n"); + // Copy the externalized provider configuration content into the topology descriptor in-line InputStreamReader policyReader = new InputStreamReader(new FileInputStream(providerConfig)); char[] buffer = new char[1024]; @@ -243,6 +260,7 @@ public class SimpleDescriptorHandler { return result; } + private static boolean validateURL(String serviceName, String url) { boolean result = false; http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/xml/KnoxFormatXmlTopologyRules.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/xml/KnoxFormatXmlTopologyRules.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/xml/KnoxFormatXmlTopologyRules.java index cb30769..7f9ec3e 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/xml/KnoxFormatXmlTopologyRules.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/xml/KnoxFormatXmlTopologyRules.java @@ -33,6 +33,7 @@ public class KnoxFormatXmlTopologyRules extends AbstractRulesModule { private static final String NAME_TAG = "name"; private static final String VERSION_TAG = "version"; private static final String DEFAULT_SERVICE_TAG = "path"; + private static final String GENERATED_TAG = "generated"; private static final String APPLICATION_TAG = "application"; private static final String SERVICE_TAG = "service"; private static final String ROLE_TAG = "role"; @@ -50,6 +51,7 @@ public class KnoxFormatXmlTopologyRules extends AbstractRulesModule { forPattern( ROOT_TAG + "/" + NAME_TAG ).callMethod("name").usingElementBodyAsArgument(); forPattern( ROOT_TAG + "/" + VERSION_TAG ).callMethod("version").usingElementBodyAsArgument(); forPattern( ROOT_TAG + "/" + DEFAULT_SERVICE_TAG ).callMethod("defaultService").usingElementBodyAsArgument(); + forPattern( ROOT_TAG + "/" + GENERATED_TAG ).callMethod("generated").usingElementBodyAsArgument(); forPattern( ROOT_TAG + "/" + APPLICATION_TAG ).createObject().ofType( Application.class ).then().setNext( "addApplication" ); forPattern( ROOT_TAG + "/" + APPLICATION_TAG + "/" + ROLE_TAG ).setBeanProperty(); http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-server/src/main/resources/conf/topology-v1.xsd ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/resources/conf/topology-v1.xsd b/gateway-server/src/main/resources/conf/topology-v1.xsd index 96c9ba2..936c701 100644 --- a/gateway-server/src/main/resources/conf/topology-v1.xsd +++ b/gateway-server/src/main/resources/conf/topology-v1.xsd @@ -22,6 +22,7 @@ limitations under the License. <h:element name="name" minOccurs="0" maxOccurs="1"/> <h:element name="path" minOccurs="0" maxOccurs="1"/> + <h:element name="generated" type="h:boolean" minOccurs="0"/> <h:element name="gateway" minOccurs="0" maxOccurs="1"> <h:complexType> http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java b/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java index a1fda1c..3a62980 100644 --- a/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java +++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandlerTest.java @@ -44,9 +44,12 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; +import static org.hamcrest.Matchers.hasXPath; +import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -231,6 +234,11 @@ public class SimpleDescriptorHandlerTest { // Parse the topology descriptor Document topologyXml = XmlUtils.readXml(topologyFile); + // KNOX-1105 Mark generated topology files + assertThat("Expected the \"generated\" marker element in the topology XML, with value of \"true\".", + topologyXml, + hasXPath("/topology/generated", is("true"))); + // Validate the provider configuration Document extProviderConf = XmlUtils.readXml(new ByteArrayInputStream(TEST_PROVIDER_CONFIG.getBytes())); Node gatewayNode = (Node) xpath.compile("/topology/gateway").evaluate(topologyXml, XPathConstants.NODE); http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/TopologiesResource.java ---------------------------------------------------------------------- diff --git a/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/TopologiesResource.java b/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/TopologiesResource.java index 445e85d..67a053b 100644 --- a/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/TopologiesResource.java +++ b/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/TopologiesResource.java @@ -135,16 +135,33 @@ public class TopologiesResource { @Consumes({APPLICATION_JSON, APPLICATION_XML}) @Path(SINGLE_TOPOLOGY_API_PATH) public Topology uploadTopology(@PathParam("id") String id, Topology t) { + Topology result = null; - GatewayServices gs = (GatewayServices) request.getServletContext() - .getAttribute(GatewayServices.GATEWAY_SERVICES_ATTRIBUTE); + GatewayServices gs = + (GatewayServices) request.getServletContext().getAttribute(GatewayServices.GATEWAY_SERVICES_ATTRIBUTE); t.setName(id); TopologyService ts = gs.getService(GatewayServices.TOPOLOGY_SERVICE); - ts.deployTopology(BeanConverter.getTopology(t)); + // Check for existing topology with the same name, to see if it had been generated + boolean existingGenerated = false; + for (org.apache.hadoop.gateway.topology.Topology existingTopology : ts.getTopologies()) { + if(existingTopology.getName().equals(id)) { + existingGenerated = existingTopology.isGenerated(); + break; + } + } - return getTopology(id); + // If a topology with the same ID exists, which had been generated, then DO NOT overwrite it because it will be + // out of sync with the source descriptor. Otherwise, deploy the updated version. + if (!existingGenerated) { + ts.deployTopology(BeanConverter.getTopology(t)); + result = getTopology(id); + } else { + log.disallowedOverwritingGeneratedTopology(id); + } + + return result; } @DELETE http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/BeanConverter.java ---------------------------------------------------------------------- diff --git a/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/BeanConverter.java b/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/BeanConverter.java index f94dcad..b25f0de 100644 --- a/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/BeanConverter.java +++ b/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/BeanConverter.java @@ -29,6 +29,7 @@ public class BeanConverter { topologyResource.setTimestamp(topology.getTimestamp()); topologyResource.setPath(topology.getDefaultServicePath()); topologyResource.setUri(topology.getUri()); + topologyResource.setGenerated(topology.isGenerated()); for ( org.apache.hadoop.gateway.topology.Provider provider : topology.getProviders() ) { topologyResource.getProviders().add( getProvider(provider) ); } @@ -47,6 +48,7 @@ public class BeanConverter { deploymentTopology.setTimestamp(topology.getTimestamp()); deploymentTopology.setDefaultServicePath(topology.getPath()); deploymentTopology.setUri(topology.getUri()); + deploymentTopology.setGenerated(topology.isGenerated()); for ( Provider provider : topology.getProviders() ) { deploymentTopology.addProvider( getProvider(provider) ); } http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/Topology.java ---------------------------------------------------------------------- diff --git a/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/Topology.java b/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/Topology.java index a197c27..67ecc27 100644 --- a/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/Topology.java +++ b/gateway-service-admin/src/main/java/org/apache/hadoop/gateway/service/admin/beans/Topology.java @@ -39,6 +39,9 @@ public class Topology { @XmlElement private long timestamp; + @XmlElement(name="generated") + private boolean isGenerated; + @XmlElement(name="provider") @XmlElementWrapper(name="gateway") public List<Provider> providers; @@ -84,6 +87,14 @@ public class Topology { this.timestamp = timestamp; } + public boolean isGenerated() { + return isGenerated; + } + + public void setGenerated(boolean isGenerated) { + this.isGenerated = isGenerated; + } + public List<Service> getServices() { if (services == null) { services = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-spi/src/main/java/org/apache/hadoop/gateway/i18n/GatewaySpiMessages.java ---------------------------------------------------------------------- diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/i18n/GatewaySpiMessages.java b/gateway-spi/src/main/java/org/apache/hadoop/gateway/i18n/GatewaySpiMessages.java index aad4d8a..4a5b926 100644 --- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/i18n/GatewaySpiMessages.java +++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/i18n/GatewaySpiMessages.java @@ -88,4 +88,7 @@ public interface GatewaySpiMessages { @Message(level = MessageLevel.ERROR, text = "Invalid resource URI {0} : {1}") void invalidResourceURI(final String uri, final String reason, @StackTrace(level = MessageLevel.DEBUG) Exception e ); + @Message( level = MessageLevel.ERROR, text = "Topology {0} cannot be manually overwritten because it was generated from a simple descriptor." ) + void disallowedOverwritingGeneratedTopology(final String topologyName); + } http://git-wip-us.apache.org/repos/asf/knox/blob/f52dc201/gateway-spi/src/main/java/org/apache/hadoop/gateway/topology/Topology.java ---------------------------------------------------------------------- diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/topology/Topology.java b/gateway-spi/src/main/java/org/apache/hadoop/gateway/topology/Topology.java index c366421..6415afe 100644 --- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/topology/Topology.java +++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/topology/Topology.java @@ -33,6 +33,7 @@ public class Topology { private String name; private String defaultServicePath = null; private long timestamp; + private boolean isGenerated; public List<Provider> providerList = new ArrayList<Provider>(); private Map<String,Map<String,Provider>> providerMap = new HashMap<>(); public List<Service> services = new ArrayList<Service>(); @@ -76,6 +77,14 @@ public class Topology { defaultServicePath = servicePath; } + public void setGenerated(boolean isGenerated) { + this.isGenerated = isGenerated; + } + + public boolean isGenerated() { + return isGenerated; + } + public Collection<Service> getServices() { return services; }
