Merge branch 'master' into KNOX-998-Package_Restructuring # Conflicts: # gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java # gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java # gateway-provider-security-pac4j/src/main/java/org/apache/knox/gateway/pac4j/filter/Pac4jIdentityAdapter.java # gateway-server/src/main/java/org/apache/knox/gateway/services/CLIGatewayServices.java # gateway-server/src/main/java/org/apache/knox/gateway/services/DefaultGatewayServices.java # gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java # gateway-server/src/main/java/org/apache/knox/gateway/topology/simple/SimpleDescriptorHandler.java # gateway-server/src/main/java/org/apache/knox/gateway/util/KnoxCLI.java # gateway-server/src/test/java/org/apache/knox/gateway/util/KnoxCLITest.java # gateway-service-definitions/src/main/resources/services/ambariui/2.2.1/service.xml
Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/22a7304a Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/22a7304a Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/22a7304a Branch: refs/heads/KNOX-998-Package_Restructuring Commit: 22a7304a602105ed01ed37e19cba086f3296e4af Parents: e70904b 370c861 Author: Sandeep More <[email protected]> Authored: Thu Dec 14 10:38:39 2017 -0500 Committer: Sandeep More <[email protected]> Committed: Thu Dec 14 10:38:39 2017 -0500 ---------------------------------------------------------------------- .gitignore | 1 + CHANGES | 22 +- gateway-admin-ui/README.md | 34 +- gateway-admin-ui/angular-cli.json | 22 +- gateway-admin-ui/package.json | 60 +- gateway-admin-ui/src/app/app.module.ts | 6 +- .../src/app/topology-detail.component.ts | 55 +- gateway-admin-ui/src/app/topology.component.ts | 2 +- .../src/assets/knox-logo-transparent.gif | Bin 0 -> 19703 bytes gateway-admin-ui/src/index.html | 4 +- gateway-admin-ui/src/tsconfig.json | 4 +- .../app/assets/knox-logo-transparent.gif | Bin 0 -> 19703 bytes .../applications/admin-ui/app/index.html | 62 +- .../app/inline.b47d11937c275f76ce02.bundle.js | 1 + .../app/inline.d41d8cd98f00b204e980.bundle.js | 2 - .../app/inline.d41d8cd98f00b204e980.bundle.map | 1 - .../app/main.806d67070af66e18c2fc.bundle.js | 2 - .../app/main.806d67070af66e18c2fc.bundle.js.gz | Bin 3657 -> 0 bytes .../app/main.806d67070af66e18c2fc.bundle.map | 1 - .../app/main.a69408978854e3a77fb2.bundle.js | 1 + .../app/scripts.2c89ed78f648df44c10f.bundle.js | 12 + .../app/styles.b2328beb0372c051d06d.bundle.js | 2 - .../app/styles.b2328beb0372c051d06d.bundle.map | 1 - .../app/styles.d41d8cd98f00b204e980.bundle.css | 0 ....d41d8cd98f00b204e9800998ecf8427e.bundle.css | 2 - .../app/vendor.48771018d3da89d3269f.bundle.js | 2035 ------------------ .../vendor.48771018d3da89d3269f.bundle.js.gz | Bin 459997 -> 0 bytes .../app/vendor.48771018d3da89d3269f.bundle.map | 1 - .../discovery/ambari/AmbariClientCommon.java | 102 + ...bariClusterConfigurationMonitorProvider.java | 35 + .../ambari/AmbariConfigurationMonitor.java | 525 +++++ .../topology/discovery/ambari/RESTInvoker.java | 136 ++ .../discovery/ambari/AmbariCluster.java | 5 + .../ambari/AmbariServiceDiscovery.java | 228 +- .../ambari/AmbariServiceDiscoveryMessages.java | 51 +- .../ambari/ServiceURLPropertyConfig.java | 2 +- ...iscovery.ClusterConfigurationMonitorProvider | 19 + .../ambari/AmbariConfigurationMonitorTest.java | 319 +++ .../ambari/AmbariServiceDiscoveryTest.java | 34 +- .../pac4j/filter/Pac4jDispatcherFilter.java | 11 +- .../pac4j/filter/Pac4jIdentityAdapter.java | 33 +- .../gateway/pac4j/MockHttpServletRequest.java | 8 +- .../knox/gateway/pac4j/Pac4jProviderTest.java | 187 +- gateway-release/home/conf/gateway-site.xml | 12 + gateway-release/home/conf/topologies/admin.xml | 21 +- .../home/conf/topologies/knoxsso.xml | 5 +- .../home/conf/topologies/manager.xml | 21 +- .../home/conf/topologies/sandbox.xml | 21 +- gateway-server/pom.xml | 9 + ...faultClusterConfigurationMonitorService.java | 81 + .../DefaultConfigurationMonitorProvider.java | 31 + .../DefaultRemoteConfigurationMonitor.java | 228 ++ .../RemoteConfigurationMonitorFactory.java | 74 + .../apache/knox/gateway/GatewayMessages.java | 64 +- .../gateway/config/impl/GatewayConfigImpl.java | 67 +- .../gateway/services/CLIGatewayServices.java | 10 + .../services/DefaultGatewayServices.java | 24 +- .../topology/impl/DefaultTopologyService.java | 99 +- .../simple/SimpleDescriptorFactory.java | 2 +- .../simple/SimpleDescriptorHandler.java | 78 +- .../simple/SimpleDescriptorMessages.java | 9 + .../org/apache/knox/gateway/util/KnoxCLI.java | 411 +++- ...y.monitor.RemoteConfigurationMonitorProvider | 19 + ...emoteConfigurationRegistryClientService.java | 263 +++ ...figurationRegistryClientServiceProvider.java | 32 + .../ZooKeeperConfigurationMonitorTest.java | 355 +++ .../config/impl/GatewayConfigImplTest.java | 43 + .../topology/DefaultTopologyServiceTest.java | 10 +- .../simple/SimpleDescriptorFactoryTest.java | 13 +- .../apache/knox/gateway/util/KnoxCLITest.java | 385 +++- .../knox/gateway/websockets/BadUrlTest.java | 11 + .../gateway/websockets/WebsocketEchoTest.java | 11 + .../WebsocketMultipleConnectionTest.java | 11 + ...teConfigurationRegistryClientServiceProvider | 19 + .../services/ambariui/2.2.1/rewrite.xml | 104 - .../services/ambariui/2.2.1/service.xml | 92 - gateway-service-remoteconfig/pom.xml | 89 + .../remote/RemoteConfigurationMessages.java | 49 + ...nfigurationRegistryClientServiceFactory.java | 41 + ...figurationRegistryClientServiceProvider.java | 27 + .../RemoteConfigurationRegistryConfig.java | 43 + .../DefaultRemoteConfigurationRegistries.java | 104 + .../config/RemoteConfigurationRegistries.java | 33 + .../RemoteConfigurationRegistriesAccessor.java | 60 + .../RemoteConfigurationRegistriesParser.java | 48 + .../config/RemoteConfigurationRegistry.java | 139 ++ .../config/remote/zk/CuratorClientService.java | 464 ++++ .../RemoteConfigurationRegistryJAASConfig.java | 179 ++ .../remote/zk/ZooKeeperClientService.java | 25 + .../zk/ZooKeeperClientServiceProvider.java | 34 + ...teConfigurationRegistryClientServiceProvider | 19 + ...efaultRemoteConfigurationRegistriesTest.java | 184 ++ ...teConfigurationRegistryConfigParserTest.java | 108 + .../util/RemoteRegistryConfigTestUtils.java | 117 + ...eConfigurationRegistryClientServiceTest.java | 424 ++++ ...moteConfigurationRegistryJAASConfigTest.java | 255 +++ .../RemoteConfigurationRegistryClient.java | 80 + ...emoteConfigurationRegistryClientService.java | 28 + .../ClusterConfigurationMonitorService.java | 43 + .../discovery/ClusterConfigurationMonitor.java | 48 + .../ClusterConfigurationMonitorProvider.java | 27 + .../monitor/RemoteConfigurationMonitor.java | 24 + .../RemoteConfigurationMonitorProvider.java | 34 + .../knox/gateway/config/GatewayConfig.java | 50 + .../knox/gateway/services/GatewayServices.java | 4 + .../apache/knox/gateway/GatewayTestConfig.java | 38 +- .../java/org/apache/knox/test/TestUtils.java | 2 +- gateway-test/pom.xml | 6 + .../SimpleDescriptorHandlerFuncTest.java | 275 +++ .../monitor/RemoteConfigurationMonitorTest.java | 603 ++++++ .../knox/gateway/GatewayBasicFuncTest.java | 2 +- ...eway.topology.discovery.ServiceDiscoveryType | 19 + pom.xml | 18 +- 113 files changed, 7743 insertions(+), 2663 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/22a7304a/.gitignore ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/22a7304a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java ---------------------------------------------------------------------- diff --cc gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java index d71e079,0000000..bcf3adc mode 100644,000000..100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java @@@ -1,115 -1,0 +1,120 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.knox.gateway.topology.discovery.ambari; + +import org.apache.knox.gateway.topology.discovery.ServiceDiscovery; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class AmbariCluster implements ServiceDiscovery.Cluster { + + private String name = null; + + private AmbariDynamicServiceURLCreator urlCreator; + + private Map<String, Map<String, ServiceConfiguration>> serviceConfigurations = new HashMap<>(); + + private Map<String, AmbariComponent> components = null; + + + AmbariCluster(String name) { + this.name = name; + components = new HashMap<>(); + urlCreator = new AmbariDynamicServiceURLCreator(this); + } + + void addServiceConfiguration(String serviceName, String configurationType, ServiceConfiguration serviceConfig) { + if (!serviceConfigurations.keySet().contains(serviceName)) { + serviceConfigurations.put(serviceName, new HashMap<>()); + } + serviceConfigurations.get(serviceName).put(configurationType, serviceConfig); + } + + + void addComponent(AmbariComponent component) { + components.put(component.getName(), component); + } + + + ServiceConfiguration getServiceConfiguration(String serviceName, String configurationType) { + ServiceConfiguration sc = null; + Map<String, ServiceConfiguration> configs = serviceConfigurations.get(serviceName); + if (configs != null) { + sc = configs.get(configurationType); + } + return sc; + } + + ++ Map<String, Map<String, ServiceConfiguration>> getServiceConfigurations() { ++ return serviceConfigurations; ++ } ++ ++ + Map<String, AmbariComponent> getComponents() { + return components; + } + + + AmbariComponent getComponent(String name) { + return components.get(name); + } + + + @Override + public String getName() { + return name; + } + + + @Override + public List<String> getServiceURLs(String serviceName) { + List<String> urls = new ArrayList<>(); + urls.addAll(urlCreator.create(serviceName)); + return urls; + } + + + static class ServiceConfiguration { + + private String type; + private String version; + private Map<String, String> props; + + ServiceConfiguration(String type, String version, Map<String, String> properties) { + this.type = type; + this.version = version; + this.props = properties; + } + + public String getVersion() { + return version; + } + + public String getType() { + return type; + } + + public Map<String, String> getProperties() { + return props; + } + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/22a7304a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java ---------------------------------------------------------------------- diff --cc gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java index dbc783d,0000000..6a6a888 mode 100644,000000..100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java @@@ -1,306 -1,0 +1,262 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.knox.gateway.topology.discovery.ambari; + - import java.io.IOException; ++import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import net.minidev.json.JSONArray; +import net.minidev.json.JSONObject; - import net.minidev.json.JSONValue; - import org.apache.knox.gateway.config.ConfigurationException; +import org.apache.knox.gateway.i18n.messages.MessagesFactory; ++import org.apache.knox.gateway.services.GatewayServices; +import org.apache.knox.gateway.services.security.AliasService; - import org.apache.knox.gateway.services.security.AliasServiceException; ++import org.apache.knox.gateway.topology.ClusterConfigurationMonitorService; ++import org.apache.knox.gateway.topology.discovery.ClusterConfigurationMonitor; +import org.apache.knox.gateway.topology.discovery.GatewayService; +import org.apache.knox.gateway.topology.discovery.ServiceDiscovery; +import org.apache.knox.gateway.topology.discovery.ServiceDiscoveryConfig; - import org.apache.http.HttpEntity; - import org.apache.http.HttpStatus; - import org.apache.http.client.methods.CloseableHttpResponse; - import org.apache.http.client.methods.HttpGet; - import org.apache.http.impl.client.CloseableHttpClient; - import org.apache.http.message.BasicHeader; - import org.apache.http.util.EntityUtils; + + +class AmbariServiceDiscovery implements ServiceDiscovery { + + static final String TYPE = "AMBARI"; + - static final String AMBARI_CLUSTERS_URI = "/api/v1/clusters"; ++ static final String AMBARI_CLUSTERS_URI = AmbariClientCommon.AMBARI_CLUSTERS_URI; + - static final String AMBARI_HOSTROLES_URI = - AMBARI_CLUSTERS_URI + "/%s/services?fields=components/host_components/HostRoles"; ++ static final String AMBARI_HOSTROLES_URI = AmbariClientCommon.AMBARI_HOSTROLES_URI; + - static final String AMBARI_SERVICECONFIGS_URI = - AMBARI_CLUSTERS_URI + "/%s/configurations/service_config_versions?is_current=true"; ++ static final String AMBARI_SERVICECONFIGS_URI = AmbariClientCommon.AMBARI_SERVICECONFIGS_URI; + + private static final String COMPONENT_CONFIG_MAPPING_FILE = + "ambari-service-discovery-component-config-mapping.properties"; + ++ private static final String GATEWAY_SERVICES_ACCESSOR_CLASS = "org.apache.knox.gateway.GatewayServer"; ++ private static final String GATEWAY_SERVICES_ACCESSOR_METHOD = "getGatewayServices"; ++ + private static final AmbariServiceDiscoveryMessages log = MessagesFactory.get(AmbariServiceDiscoveryMessages.class); + + // Map of component names to service configuration types + private static Map<String, String> componentServiceConfigs = new HashMap<>(); + static { + try { + Properties configMapping = new Properties(); + configMapping.load(AmbariServiceDiscovery.class.getClassLoader().getResourceAsStream(COMPONENT_CONFIG_MAPPING_FILE)); + for (String componentName : configMapping.stringPropertyNames()) { + componentServiceConfigs.put(componentName, configMapping.getProperty(componentName)); + } + } catch (Exception e) { - log.failedToLoadServiceDiscoveryConfiguration(COMPONENT_CONFIG_MAPPING_FILE, e); ++ log.failedToLoadServiceDiscoveryURLDefConfiguration(COMPONENT_CONFIG_MAPPING_FILE, e); + } + } + - private static final String DEFAULT_USER_ALIAS = "ambari.discovery.user"; - private static final String DEFAULT_PWD_ALIAS = "ambari.discovery.password"; - + @GatewayService + private AliasService aliasService; + - private CloseableHttpClient httpClient = null; ++ private RESTInvoker restClient; ++ private AmbariClientCommon ambariClient; + ++ // This is used to update the monitor when new cluster configuration details are discovered. ++ private AmbariConfigurationMonitor configChangeMonitor; ++ ++ private boolean isInitialized = false; + + AmbariServiceDiscovery() { - httpClient = org.apache.http.impl.client.HttpClients.createDefault(); ++ } ++ ++ ++ AmbariServiceDiscovery(RESTInvoker restClient) { ++ this.restClient = restClient; ++ } ++ ++ ++ /** ++ * Initialization must be subsequent to construction because the AliasService member isn't assigned until after ++ * construction time. This is called internally prior to discovery invocations to make sure the clients have been ++ * initialized. ++ */ ++ private void init() { ++ if (!isInitialized) { ++ if (this.restClient == null) { ++ this.restClient = new RESTInvoker(aliasService); ++ } ++ this.ambariClient = new AmbariClientCommon(restClient); ++ this.configChangeMonitor = getConfigurationChangeMonitor(); ++ ++ isInitialized = true; ++ } ++ } ++ ++ ++ /** ++ * Get the Ambari configuration change monitor from the associated gateway service. ++ */ ++ private AmbariConfigurationMonitor getConfigurationChangeMonitor() { ++ AmbariConfigurationMonitor ambariMonitor = null; ++ try { ++ Class clazz = Class.forName(GATEWAY_SERVICES_ACCESSOR_CLASS); ++ if (clazz != null) { ++ Method m = clazz.getDeclaredMethod(GATEWAY_SERVICES_ACCESSOR_METHOD); ++ if (m != null) { ++ Object obj = m.invoke(null); ++ if (GatewayServices.class.isAssignableFrom(obj.getClass())) { ++ ClusterConfigurationMonitorService clusterMonitorService = ++ ((GatewayServices) obj).getService(GatewayServices.CLUSTER_CONFIGURATION_MONITOR_SERVICE); ++ ClusterConfigurationMonitor monitor = ++ clusterMonitorService.getMonitor(AmbariConfigurationMonitor.getType()); ++ if (monitor != null) { ++ if (AmbariConfigurationMonitor.class.isAssignableFrom(monitor.getClass())) { ++ ambariMonitor = (AmbariConfigurationMonitor) monitor; ++ } ++ } ++ } ++ } ++ } ++ } catch (Exception e) { ++ log.errorAccessingConfigurationChangeMonitor(e); ++ } ++ return ambariMonitor; + } + + + @Override + public String getType() { + return TYPE; + } + + + @Override + public Map<String, Cluster> discover(ServiceDiscoveryConfig config) { - Map<String, Cluster> clusters = new HashMap<String, Cluster>(); ++ Map<String, Cluster> clusters = new HashMap<>(); ++ ++ init(); + + String discoveryAddress = config.getAddress(); + + // Invoke Ambari REST API to discover the available clusters + String clustersDiscoveryURL = String.format("%s" + AMBARI_CLUSTERS_URI, discoveryAddress); + - JSONObject json = invokeREST(clustersDiscoveryURL, config.getUser(), config.getPasswordAlias()); ++ JSONObject json = restClient.invoke(clustersDiscoveryURL, config.getUser(), config.getPasswordAlias()); + + // Parse the cluster names from the response, and perform the cluster discovery + JSONArray clusterItems = (JSONArray) json.get("items"); + for (Object clusterItem : clusterItems) { + String clusterName = (String) ((JSONObject)((JSONObject) clusterItem).get("Clusters")).get("cluster_name"); + try { + Cluster c = discover(config, clusterName); + clusters.put(clusterName, c); + } catch (Exception e) { + log.clusterDiscoveryError(clusterName, e); + } + } + + return clusters; + } + + + @Override + public Cluster discover(ServiceDiscoveryConfig config, String clusterName) { + AmbariCluster cluster = new AmbariCluster(clusterName); + + Map<String, String> serviceComponents = new HashMap<>(); + ++ init(); ++ + String discoveryAddress = config.getAddress(); + String discoveryUser = config.getUser(); + String discoveryPwdAlias = config.getPasswordAlias(); + + Map<String, List<String>> componentHostNames = new HashMap<>(); + String hostRolesURL = String.format("%s" + AMBARI_HOSTROLES_URI, discoveryAddress, clusterName); - JSONObject hostRolesJSON = invokeREST(hostRolesURL, discoveryUser, discoveryPwdAlias); ++ JSONObject hostRolesJSON = restClient.invoke(hostRolesURL, discoveryUser, discoveryPwdAlias); + if (hostRolesJSON != null) { + // Process the host roles JSON + JSONArray items = (JSONArray) hostRolesJSON.get("items"); + for (Object obj : items) { + JSONArray components = (JSONArray) ((JSONObject) obj).get("components"); + for (Object component : components) { + JSONArray hostComponents = (JSONArray) ((JSONObject) component).get("host_components"); + for (Object hostComponent : hostComponents) { + JSONObject hostRoles = (JSONObject) ((JSONObject) hostComponent).get("HostRoles"); + String serviceName = (String) hostRoles.get("service_name"); + String componentName = (String) hostRoles.get("component_name"); + + serviceComponents.put(componentName, serviceName); + + // Assuming public host name is more applicable than host_name + String hostName = (String) hostRoles.get("public_host_name"); + if (hostName == null) { + // Some (even slightly) older versions of Ambari/HDP do not return public_host_name, + // so fall back to host_name in those cases. + hostName = (String) hostRoles.get("host_name"); + } + + if (hostName != null) { + log.discoveredServiceHost(serviceName, hostName); + if (!componentHostNames.containsKey(componentName)) { - componentHostNames.put(componentName, new ArrayList<String>()); ++ componentHostNames.put(componentName, new ArrayList<>()); + } + componentHostNames.get(componentName).add(hostName); + } + } + } + } + } + ++ // Service configurations + Map<String, Map<String, AmbariCluster.ServiceConfiguration>> serviceConfigurations = - new HashMap<String, Map<String, AmbariCluster.ServiceConfiguration>>(); - String serviceConfigsURL = String.format("%s" + AMBARI_SERVICECONFIGS_URI, discoveryAddress, clusterName); - JSONObject serviceConfigsJSON = invokeREST(serviceConfigsURL, discoveryUser, discoveryPwdAlias); - if (serviceConfigsJSON != null) { - // Process the service configurations - JSONArray serviceConfigs = (JSONArray) serviceConfigsJSON.get("items"); - for (Object serviceConfig : serviceConfigs) { - String serviceName = (String) ((JSONObject) serviceConfig).get("service_name"); - JSONArray configurations = (JSONArray) ((JSONObject) serviceConfig).get("configurations"); - for (Object configuration : configurations) { - String configType = (String) ((JSONObject) configuration).get("type"); - String configVersion = String.valueOf(((JSONObject) configuration).get("version")); - - Map<String, String> configProps = new HashMap<String, String>(); - JSONObject configProperties = (JSONObject) ((JSONObject) configuration).get("properties"); - for (String propertyName : configProperties.keySet()) { - configProps.put(propertyName, String.valueOf(((JSONObject) configProperties).get(propertyName))); - } - if (!serviceConfigurations.containsKey(serviceName)) { - serviceConfigurations.put(serviceName, new HashMap<String, AmbariCluster.ServiceConfiguration>()); - } - serviceConfigurations.get(serviceName).put(configType, new AmbariCluster.ServiceConfiguration(configType, configVersion, configProps)); - cluster.addServiceConfiguration(serviceName, configType, new AmbariCluster.ServiceConfiguration(configType, configVersion, configProps)); - } ++ ambariClient.getActiveServiceConfigurations(discoveryAddress, ++ clusterName, ++ discoveryUser, ++ discoveryPwdAlias); ++ for (String serviceName : serviceConfigurations.keySet()) { ++ for (Map.Entry<String, AmbariCluster.ServiceConfiguration> serviceConfig : serviceConfigurations.get(serviceName).entrySet()) { ++ cluster.addServiceConfiguration(serviceName, serviceConfig.getKey(), serviceConfig.getValue()); + } + } + + // Construct the AmbariCluster model + for (String componentName : serviceComponents.keySet()) { + String serviceName = serviceComponents.get(componentName); + List<String> hostNames = componentHostNames.get(componentName); + + Map<String, AmbariCluster.ServiceConfiguration> configs = serviceConfigurations.get(serviceName); + String configType = componentServiceConfigs.get(componentName); + if (configType != null) { + AmbariCluster.ServiceConfiguration svcConfig = configs.get(configType); + AmbariComponent c = new AmbariComponent(componentName, + svcConfig.getVersion(), + clusterName, + serviceName, + hostNames, + svcConfig.getProperties()); + cluster.addComponent(c); + } + } + - return cluster; - } - - - protected JSONObject invokeREST(String url, String username, String passwordAlias) { - JSONObject result = null; - - CloseableHttpResponse response = null; - try { - HttpGet request = new HttpGet(url); - - // If no configured username, then use default username alias - String password = null; - if (username == null) { - if (aliasService != null) { - try { - char[] defaultUser = aliasService.getPasswordFromAliasForGateway(DEFAULT_USER_ALIAS); - if (defaultUser != null) { - username = new String(defaultUser); - } - } catch (AliasServiceException e) { - log.aliasServiceUserError(DEFAULT_USER_ALIAS, e.getLocalizedMessage()); - } - } - - // If username is still null - if (username == null) { - log.aliasServiceUserNotFound(); - throw new ConfigurationException("No username is configured for Ambari service discovery."); - } - } - - if (aliasService != null) { - // If no password alias is configured, then try the default alias - if (passwordAlias == null) { - passwordAlias = DEFAULT_PWD_ALIAS; - } - - try { - char[] pwd = aliasService.getPasswordFromAliasForGateway(passwordAlias); - if (pwd != null) { - password = new String(pwd); - } - - } catch (AliasServiceException e) { - log.aliasServicePasswordError(passwordAlias, e.getLocalizedMessage()); - } - } - - // If the password could not be determined - if (password == null) { - log.aliasServicePasswordNotFound(); - throw new ConfigurationException("No password is configured for Ambari service discovery."); - } - - // Add an auth header if credentials are available - String encodedCreds = - org.apache.commons.codec.binary.Base64.encodeBase64String((username + ":" + password).getBytes()); - request.addHeader(new BasicHeader("Authorization", "Basic " + encodedCreds)); - - response = httpClient.execute(request); - - if (HttpStatus.SC_OK == response.getStatusLine().getStatusCode()) { - HttpEntity entity = response.getEntity(); - if (entity != null) { - result = (JSONObject) JSONValue.parse((EntityUtils.toString(entity))); - log.debugJSON(result.toJSONString()); - } else { - log.noJSON(url); - } - } else { - log.unexpectedRestResponseStatusCode(url, response.getStatusLine().getStatusCode()); - } - - } catch (IOException e) { - log.restInvocationError(url, e); - } finally { - if(response != null) { - try { - response.close(); - } catch (IOException e) { - // Ignore - } - } ++ if (configChangeMonitor != null) { ++ // Notify the cluster config monitor about these cluster configuration details ++ configChangeMonitor.addClusterConfigVersions(cluster, config); + } - return result; - } + ++ return cluster; ++ } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/22a7304a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java ---------------------------------------------------------------------- diff --cc gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java index 2bdc94b,0000000..12e6078 mode 100644,000000..100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java @@@ -1,121 -1,0 +1,148 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.knox.gateway.topology.discovery.ambari; + +import org.apache.knox.gateway.i18n.messages.Message; +import org.apache.knox.gateway.i18n.messages.MessageLevel; +import org.apache.knox.gateway.i18n.messages.Messages; +import org.apache.knox.gateway.i18n.messages.StackTrace; + +@Messages(logger="org.apache.knox.gateway.topology.discovery.ambari") +public interface AmbariServiceDiscoveryMessages { + + @Message(level = MessageLevel.ERROR, - text = "Failed to load service discovery configuration: {1}") - void failedToLoadServiceDiscoveryConfiguration(@StackTrace(level = MessageLevel.ERROR) Exception e); ++ text = "Failed to persist data for cluster configuration monitor {0} {1}: {2}") ++ void failedToPersistClusterMonitorData(final String monitor, ++ final String filename, ++ @StackTrace(level = MessageLevel.DEBUG) Exception e); + + @Message(level = MessageLevel.ERROR, - text = "Failed to load service discovery configuration {0}: {1}") - void failedToLoadServiceDiscoveryConfiguration(final String configuration, - @StackTrace(level = MessageLevel.ERROR) Exception e); ++ text = "Failed to load persisted service discovery configuration for cluster monitor {0} : {1}") ++ void failedToLoadClusterMonitorServiceDiscoveryConfig(final String monitor, ++ @StackTrace(level = MessageLevel.DEBUG) Exception e); ++ ++ @Message(level = MessageLevel.ERROR, ++ text = "Failed to load persisted cluster configuration version data for cluster monitor {0} : {1}") ++ void failedToLoadClusterMonitorConfigVersions(final String monitor, ++ @StackTrace(level = MessageLevel.DEBUG) Exception e); ++ ++ @Message(level = MessageLevel.ERROR, ++ text = "Unable to access the Ambari Configuration Change Monitor: {0}") ++ void errorAccessingConfigurationChangeMonitor(@StackTrace(level = MessageLevel.DEBUG) Exception e); ++ ++ @Message(level = MessageLevel.ERROR, ++ text = "Failed to load service discovery URL definition configuration: {1}") ++ void failedToLoadServiceDiscoveryURLDefConfiguration(@StackTrace(level = MessageLevel.DEBUG) Exception e); ++ ++ @Message(level = MessageLevel.ERROR, ++ text = "Failed to load service discovery URL definition configuration {0}: {1}") ++ void failedToLoadServiceDiscoveryURLDefConfiguration(final String configuration, ++ @StackTrace(level = MessageLevel.ERROR) Exception e); + + @Message(level = MessageLevel.ERROR, + text = "Encountered an error during cluster {0} discovery: {1}") + void clusterDiscoveryError(final String clusterName, - @StackTrace(level = MessageLevel.ERROR) Exception e); ++ @StackTrace(level = MessageLevel.DEBUG) Exception e); + + + @Message(level = MessageLevel.DEBUG, + text = "REST invocation {0} failed: {1}") + void restInvocationError(final String url, - @StackTrace(level = MessageLevel.ERROR) Exception e); ++ @StackTrace(level = MessageLevel.DEBUG) Exception e); + + + @Message(level = MessageLevel.ERROR, + text = "Encountered an error attempting to determine the user for alias {0} : {1}") + void aliasServiceUserError(final String alias, final String error); + + + @Message(level = MessageLevel.ERROR, + text = "Encountered an error attempting to determine the password for alias {0} : {1}") + void aliasServicePasswordError(final String alias, final String error); + + + @Message(level = MessageLevel.ERROR, + text = "No user configured for Ambari service discovery.") + void aliasServiceUserNotFound(); + + + @Message(level = MessageLevel.ERROR, + text = "No password configured for Ambari service discovery.") + void aliasServicePasswordNotFound(); + + + @Message(level = MessageLevel.ERROR, + text = "Unexpected REST invocation response code for {0} : {1}") + void unexpectedRestResponseStatusCode(final String url, int responseStatusCode); + + + @Message(level = MessageLevel.ERROR, + text = "REST invocation {0} yielded a response without any JSON.") + void noJSON(final String url); + + - @Message(level = MessageLevel.DEBUG, ++ @Message(level = MessageLevel.TRACE, + text = "REST invocation result: {0}") + void debugJSON(final String json); + ++ + @Message(level = MessageLevel.DEBUG, - text = "Loaded component configuration mappings: {0}") ++ text = "Loaded component configuration mappings: {0}") + void loadedComponentConfigMappings(final String mappings); + ++ + @Message(level = MessageLevel.ERROR, + text = "Failed to load component configuration property mappings {0}: {1}") + void failedToLoadComponentConfigMappings(final String mappings, - @StackTrace(level = MessageLevel.ERROR) Exception e); ++ @StackTrace(level = MessageLevel.DEBUG) Exception e); + - @Message(level = MessageLevel.DEBUG, ++ ++ @Message(level = MessageLevel.TRACE, + text = "Discovered: Service: {0}, Host: {1}") + void discoveredServiceHost(final String serviceName, final String hostName); + + + @Message(level = MessageLevel.DEBUG, + text = "Querying the cluster for the {0} configuration ({1}) property: {2}") + void lookingUpServiceConfigProperty(final String serviceName, final String configType, final String propertyName); + + + @Message(level = MessageLevel.DEBUG, + text = "Querying the cluster for the {0} component configuration property: {1}") + void lookingUpComponentConfigProperty(final String componentName, final String propertyName); + + + @Message(level = MessageLevel.DEBUG, + text = "Querying the cluster for the {0} component's hosts") + void lookingUpComponentHosts(final String componentName); + + + @Message(level = MessageLevel.DEBUG, + text = "Handling a derived service URL mapping property for the {0} service: type = {1}, name = {2}") + void handlingDerivedProperty(final String serviceName, final String propertyType, final String propertyName); + + + @Message(level = MessageLevel.DEBUG, - text = "Determined the service URL mapping property {0} value: {1}") ++ text = "Determined the service URL mapping property {0} value: {1}") + void determinedPropertyValue(final String propertyName, final String propertyValue); + + ++ @Message(level = MessageLevel.INFO, ++ text = "Started Ambari cluster configuration monitor (checking every {0} seconds)") ++ void startedAmbariConfigMonitor(final long pollingInterval); ++ +} http://git-wip-us.apache.org/repos/asf/knox/blob/22a7304a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLPropertyConfig.java ---------------------------------------------------------------------- diff --cc gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLPropertyConfig.java index ed07873,0000000..47b20e9 mode 100644,000000..100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLPropertyConfig.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/ServiceURLPropertyConfig.java @@@ -1,324 -1,0 +1,324 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.knox.gateway.topology.discovery.ambari; + +import org.apache.knox.gateway.i18n.messages.MessagesFactory; +import org.apache.knox.gateway.util.XmlUtils; +import org.w3c.dom.Document; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Service URL pattern mapping configuration model. + */ +class ServiceURLPropertyConfig { + + private static final AmbariServiceDiscoveryMessages log = MessagesFactory.get(AmbariServiceDiscoveryMessages.class); + + private static final String ATTR_NAME = "name"; + + private static XPathExpression SERVICE_URL_PATTERN_MAPPINGS; + private static XPathExpression URL_PATTERN; + private static XPathExpression PROPERTIES; + static { + XPath xpath = XPathFactory.newInstance().newXPath(); + try { + SERVICE_URL_PATTERN_MAPPINGS = xpath.compile("/service-discovery-url-mappings/service"); + URL_PATTERN = xpath.compile("url-pattern/text()"); + PROPERTIES = xpath.compile("properties/property"); + } catch (XPathExpressionException e) { + e.printStackTrace(); + } + } + + private static final String DEFAULT_SERVICE_URL_MAPPINGS = "ambari-service-discovery-url-mappings.xml"; + + private Map<String, URLPattern> urlPatterns = new HashMap<>(); + + private Map<String, Map<String, Property>> properties = new HashMap<>(); + + + /** + * The default service URL pattern to property mapping configuration will be used. + */ + ServiceURLPropertyConfig() { + this(ServiceURLPropertyConfig.class.getClassLoader().getResourceAsStream(DEFAULT_SERVICE_URL_MAPPINGS)); + } + + /** + * The default service URL pattern to property mapping configuration will be used. + */ + ServiceURLPropertyConfig(File mappingConfigurationFile) throws Exception { + this(new FileInputStream(mappingConfigurationFile)); + } + + /** + * + * @param source An InputStream for the XML content + */ + ServiceURLPropertyConfig(InputStream source) { + // Parse the XML, and build the model + try { + Document doc = XmlUtils.readXml(source); + + NodeList serviceNodes = + (NodeList) SERVICE_URL_PATTERN_MAPPINGS.evaluate(doc, XPathConstants.NODESET); + for (int i=0; i < serviceNodes.getLength(); i++) { + Node serviceNode = serviceNodes.item(i); + String serviceName = serviceNode.getAttributes().getNamedItem(ATTR_NAME).getNodeValue(); + properties.put(serviceName, new HashMap<String, Property>()); + + Node urlPatternNode = (Node) URL_PATTERN.evaluate(serviceNode, XPathConstants.NODE); + if (urlPatternNode != null) { + urlPatterns.put(serviceName, new URLPattern(urlPatternNode.getNodeValue())); + } + + NodeList propertiesNode = (NodeList) PROPERTIES.evaluate(serviceNode, XPathConstants.NODESET); + if (propertiesNode != null) { + processProperties(serviceName, propertiesNode); + } + } + } catch (Exception e) { - log.failedToLoadServiceDiscoveryConfiguration(e); ++ log.failedToLoadServiceDiscoveryURLDefConfiguration(e); + } finally { + try { + source.close(); + } catch (IOException e) { + // Ignore + } + } + } + + private void processProperties(String serviceName, NodeList propertyNodes) { + for (int i = 0; i < propertyNodes.getLength(); i++) { + Property p = Property.createProperty(serviceName, propertyNodes.item(i)); + properties.get(serviceName).put(p.getName(), p); + } + } + + URLPattern getURLPattern(String service) { + return urlPatterns.get(service); + } + + Property getConfigProperty(String service, String property) { + return properties.get(service).get(property); + } + + static class URLPattern { + String pattern; + List<String> placeholders = new ArrayList<>(); + + URLPattern(String pattern) { + this.pattern = pattern; + + final Pattern regex = Pattern.compile("\\{(.*?)}", Pattern.DOTALL); + final Matcher matcher = regex.matcher(pattern); + while( matcher.find() ){ + placeholders.add(matcher.group(1)); + } + } + + String get() {return pattern; } + List<String> getPlaceholders() { + return placeholders; + } + } + + static class Property { + static final String TYPE_SERVICE = "SERVICE"; + static final String TYPE_COMPONENT = "COMPONENT"; + static final String TYPE_DERIVED = "DERIVED"; + + static final String PROP_COMP_HOSTNAME = "component.host.name"; + + static final String ATTR_NAME = "name"; + static final String ATTR_PROPERTY = "property"; + static final String ATTR_VALUE = "value"; + + static XPathExpression HOSTNAME; + static XPathExpression SERVICE_CONFIG; + static XPathExpression COMPONENT; + static XPathExpression CONFIG_PROPERTY; + static XPathExpression IF; + static XPathExpression THEN; + static XPathExpression ELSE; + static XPathExpression TEXT; + static { + XPath xpath = XPathFactory.newInstance().newXPath(); + try { + HOSTNAME = xpath.compile("hostname"); + SERVICE_CONFIG = xpath.compile("service-config"); + COMPONENT = xpath.compile("component"); + CONFIG_PROPERTY = xpath.compile("config-property"); + IF = xpath.compile("if"); + THEN = xpath.compile("then"); + ELSE = xpath.compile("else"); + TEXT = xpath.compile("text()"); + } catch (XPathExpressionException e) { + e.printStackTrace(); + } + } + + + String type; + String name; + String component; + String service; + String serviceConfig; + String value; + ConditionalValueHandler conditionHandler = null; + + private Property(String type, + String propertyName, + String component, + String service, + String configType, + String value, + ConditionalValueHandler pch) { + this.type = type; + this.name = propertyName; + this.service = service; + this.component = component; + this.serviceConfig = configType; + this.value = value; + conditionHandler = pch; + } + + static Property createProperty(String serviceName, Node propertyNode) { + String propertyName = propertyNode.getAttributes().getNamedItem(ATTR_NAME).getNodeValue(); + String propertyType = null; + String serviceType = null; + String configType = null; + String componentType = null; + String value = null; + ConditionalValueHandler pch = null; + + try { + Node hostNameNode = (Node) HOSTNAME.evaluate(propertyNode, XPathConstants.NODE); + if (hostNameNode != null) { + value = PROP_COMP_HOSTNAME; + } + + // Check for a service-config node + Node scNode = (Node) SERVICE_CONFIG.evaluate(propertyNode, XPathConstants.NODE); + if (scNode != null) { + // Service config property + propertyType = Property.TYPE_SERVICE; + serviceType = scNode.getAttributes().getNamedItem(ATTR_NAME).getNodeValue(); + Node scTextNode = (Node) TEXT.evaluate(scNode, XPathConstants.NODE); + configType = scTextNode.getNodeValue(); + } else { // If not service-config node, check for a component config node + Node cNode = (Node) COMPONENT.evaluate(propertyNode, XPathConstants.NODE); + if (cNode != null) { + // Component config property + propertyType = Property.TYPE_COMPONENT; + componentType = cNode.getFirstChild().getNodeValue(); + Node cTextNode = (Node) TEXT.evaluate(cNode, XPathConstants.NODE); + configType = cTextNode.getNodeValue(); + componentType = cTextNode.getNodeValue(); + } + } + + // Check for a config property node + Node cpNode = (Node) CONFIG_PROPERTY.evaluate(propertyNode, XPathConstants.NODE); + if (cpNode != null) { + // Check for a condition element + Node ifNode = (Node) IF.evaluate(cpNode, XPathConstants.NODE); + if (ifNode != null) { + propertyType = TYPE_DERIVED; + pch = getConditionHandler(serviceName, ifNode); + } else { + Node cpTextNode = (Node) TEXT.evaluate(cpNode, XPathConstants.NODE); + value = cpTextNode.getNodeValue(); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + // Create and return the property representation + return new Property(propertyType, propertyName, componentType, serviceType, configType, value, pch); + } + + private static ConditionalValueHandler getConditionHandler(String serviceName, Node ifNode) throws Exception { + ConditionalValueHandler result = null; + + if (ifNode != null) { + NamedNodeMap attrs = ifNode.getAttributes(); + String comparisonPropName = attrs.getNamedItem(ATTR_PROPERTY).getNodeValue(); + String comparisonValue = attrs.getNamedItem(ATTR_VALUE).getNodeValue(); + + ConditionalValueHandler affirmativeResult = null; + Node thenNode = (Node) THEN.evaluate(ifNode, XPathConstants.NODE); + if (thenNode != null) { + Node subIfNode = (Node) IF.evaluate(thenNode, XPathConstants.NODE); + if (subIfNode != null) { + affirmativeResult = getConditionHandler(serviceName, subIfNode); + } else { + affirmativeResult = new SimpleValueHandler(thenNode.getFirstChild().getNodeValue()); + } + } + + ConditionalValueHandler negativeResult = null; + Node elseNode = (Node) ELSE.evaluate(ifNode, XPathConstants.NODE); + if (elseNode != null) { + Node subIfNode = (Node) IF.evaluate(elseNode, XPathConstants.NODE); + if (subIfNode != null) { + negativeResult = getConditionHandler(serviceName, subIfNode); + } else { + negativeResult = new SimpleValueHandler(elseNode.getFirstChild().getNodeValue()); + } + } + + result = new PropertyEqualsHandler(serviceName, + comparisonPropName, + comparisonValue, + affirmativeResult, + negativeResult); + } + + return result; + } + + String getType() { return type; } + String getName() { return name; } + String getComponent() { return component; } + String getService() { return service; } + String getServiceConfig() { return serviceConfig; } + String getValue() { + return value; + } + ConditionalValueHandler getConditionHandler() { return conditionHandler; } + } +}
