Repository: knox Updated Branches: refs/heads/master 1a50523f3 -> 21ac567db
KNOX-1193 - Discovery support for SOLR, STORM and STORM-LOGVIEWER, and config for additional support Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/21ac567d Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/21ac567d Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/21ac567d Branch: refs/heads/master Commit: 21ac567db07c3f2196dfb41cb7bf9bbb786801a2 Parents: 1a50523 Author: Phil Zampino <pzamp...@apache.org> Authored: Fri Mar 2 15:52:06 2018 -0500 Committer: Phil Zampino <pzamp...@apache.org> Committed: Fri Mar 2 15:59:33 2018 -0500 ---------------------------------------------------------------------- .../ambari/AmbariDynamicServiceURLCreator.java | 21 +++++++++- .../ambari/AmbariServiceDiscovery.java | 21 +++++++++- ...iscovery-component-config-mapping.properties | 26 ++++++------ .../ambari-service-discovery-url-mappings.xml | 42 ++++++++++++++++++++ .../topology/impl/DefaultTopologyService.java | 6 +++ .../topology/discovery/ServiceDiscovery.java | 2 + 6 files changed, 104 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/21ac567d/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java index 96d462b..7ccd170 100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariDynamicServiceURLCreator.java @@ -17,6 +17,7 @@ package org.apache.knox.gateway.topology.discovery.ambari; import org.apache.knox.gateway.i18n.messages.MessagesFactory; +import org.apache.knox.gateway.topology.discovery.ServiceDiscovery; import java.io.ByteArrayInputStream; import java.io.File; @@ -30,6 +31,8 @@ import java.util.Map; class AmbariDynamicServiceURLCreator implements ServiceURLCreator { + static final String MAPPING_CONFIG_OVERRIDE_FILE = "ambari-discovery-url-mappings.xml"; + static final String MAPPING_CONFIG_OVERRIDE_PROPERTY = "org.apache.gateway.topology.discovery.ambari.config"; private AmbariServiceDiscoveryMessages log = MessagesFactory.get(AmbariServiceDiscoveryMessages.class); @@ -44,7 +47,23 @@ class AmbariDynamicServiceURLCreator implements ServiceURLCreator { config = new ServiceURLPropertyConfig(); // Attempt to apply overriding or additional mappings from external source - String mappingConfiguration = System.getProperty(MAPPING_CONFIG_OVERRIDE_PROPERTY); + String mappingConfiguration = null; + + // First, check for the well-known override mapping file + String gatewayConfDir = System.getProperty(ServiceDiscovery.CONFIG_DIR_PROPERTY); + if (gatewayConfDir != null) { + File overridesFile = new File(gatewayConfDir, MAPPING_CONFIG_OVERRIDE_FILE); + if (overridesFile.exists()) { + mappingConfiguration = overridesFile.getAbsolutePath(); + } + } + + // If no file in the config dir, check for the system property reference + if (mappingConfiguration == null) { + mappingConfiguration = System.getProperty(MAPPING_CONFIG_OVERRIDE_PROPERTY); + } + + // If there is an overrides configuration file specified either way, then apply it if (mappingConfiguration != null) { File mappingConfigFile = new File(mappingConfiguration); if (mappingConfigFile.exists()) { http://git-wip-us.apache.org/repos/asf/knox/blob/21ac567d/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java index 80009cc..957d723 100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java @@ -16,6 +16,7 @@ */ package org.apache.knox.gateway.topology.discovery.ambari; +import java.io.File; import java.io.FileInputStream; import java.io.InputStream; import java.lang.reflect.Method; @@ -53,6 +54,8 @@ class AmbariServiceDiscovery implements ServiceDiscovery { private static final String COMPONENT_CONFIG_MAPPING_FILE = "ambari-service-discovery-component-config-mapping.properties"; + private static final String COMPONENT_CONFIG_OVERRIDES_FILENAME = "ambari-discovery-component-config.properties"; + private static final String GATEWAY_SERVICES_ACCESSOR_CLASS = "org.apache.knox.gateway.GatewayServer"; private static final String GATEWAY_SERVICES_ACCESSOR_METHOD = "getGatewayServices"; @@ -87,7 +90,23 @@ class AmbariServiceDiscovery implements ServiceDiscovery { } // Attempt to apply overriding or additional mappings from external source - String overridesPath = System.getProperty(COMPONENT_CONFIG_MAPPING_SYSTEM_PROPERTY); + String overridesPath = null; + + // First, check for the well-known overrides config file + String gatewayConfDir = System.getProperty(CONFIG_DIR_PROPERTY); + if (gatewayConfDir != null) { + File overridesFile = new File(gatewayConfDir, COMPONENT_CONFIG_OVERRIDES_FILENAME); + if (overridesFile.exists()) { + overridesPath = overridesFile.getAbsolutePath(); + } + } + + // If no file in the config dir, check for the system property reference + if (overridesPath == null) { + overridesPath = System.getProperty(COMPONENT_CONFIG_MAPPING_SYSTEM_PROPERTY); + } + + // If there is an overrides configuration file specified either way, then apply it if (overridesPath != null) { Properties overrides = new Properties(); try (InputStream in = new FileInputStream(overridesPath)) { http://git-wip-us.apache.org/repos/asf/knox/blob/21ac567d/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties index 48dd7b1..bf53db0 100644 --- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties +++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-component-config-mapping.properties @@ -20,22 +20,24 @@ # Hadoop service component configuration mappings # Service component name = service component configuration type ########################################################################## -NAMENODE=hdfs-site -RESOURCEMANAGER=yarn-site -HISTORYSERVER=mapred-site -OOZIE_SERVER=oozie-site -HIVE_SERVER=hive-site -WEBHCAT_SERVER=webhcat-site -HBASE_MASTER=hbase-site -DRUID_COORDINATOR=druid-coordinator +ATLAS_SERVER=application-properties DRUID_BROKER=druid-broker -DRUID_ROUTER=druid-router +DRUID_COORDINATOR=druid-coordinator DRUID_OVERLORD=druid-overlord +DRUID_ROUTER=druid-router DRUID_SUPERSET=druid-superset -ATLAS_SERVER=application-properties -ZEPPELIN_MASTER=zeppelin-config -RANGER_ADMIN=ranger-admin-site FALCON_SERVER=falcon-env +HBASE_MASTER=hbase-site +HISTORYSERVER=mapred-site +HIVE_SERVER=hive-site +INFRA_SOLR=infra-solr-env LIVY_SERVER=livy-conf +NAMENODE=hdfs-site +OOZIE_SERVER=oozie-site +RANGER_ADMIN=ranger-admin-site +RESOURCEMANAGER=yarn-site SPARK_JOBHISTORYSERVER=spark-defaults +STORM_UI_SERVER=storm-site +WEBHCAT_SERVER=webhcat-site +ZEPPELIN_MASTER=zeppelin-config http://git-wip-us.apache.org/repos/asf/knox/blob/21ac567d/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml index 6b6907d..3f37c0a 100644 --- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml +++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-url-mappings.xml @@ -568,4 +568,46 @@ </properties> </service> + <service name="SOLR"> + <url-pattern>http://{HOST}:{PORT}</url-pattern> + <properties> + <property name="HOST"> + <component>INFRA_SOLR</component> + <hostname/> + </property> + <property name="PORT"> + <component>INFRA_SOLR</component> + <config-property>infra_solr_port</config-property> + </property> + </properties> + </service> + + <service name="STORM"> + <url-pattern>http://{HOST}:{PORT}</url-pattern> + <properties> + <property name="HOST"> + <component>STORM_UI_SERVER</component> + <hostname/> + </property> + <property name="PORT"> + <component>STORM_UI_SERVER</component> + <config-property>ui.port</config-property> + </property> + </properties> + </service> + + <service name="STORM-LOGVIEWER"> + <url-pattern>http://{HOST}:{PORT}</url-pattern> + <properties> + <property name="HOST"> + <component>STORM_UI_SERVER</component> + <hostname/> + </property> + <property name="PORT"> + <component>STORM_UI_SERVER</component> + <config-property>logviewer.port</config-property> + </property> + </properties> + </service> + </service-discovery-url-mappings> http://git-wip-us.apache.org/repos/asf/knox/blob/21ac567d/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java b/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java index 52a7dfa..ccd4130 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/services/topology/impl/DefaultTopologyService.java @@ -51,6 +51,7 @@ import org.apache.knox.gateway.topology.TopologyMonitor; import org.apache.knox.gateway.topology.TopologyProvider; import org.apache.knox.gateway.topology.builder.TopologyBuilder; import org.apache.knox.gateway.topology.discovery.ClusterConfigurationMonitor; +import org.apache.knox.gateway.topology.discovery.ServiceDiscovery; import org.apache.knox.gateway.topology.monitor.RemoteConfigurationMonitor; import org.apache.knox.gateway.topology.monitor.RemoteConfigurationMonitorFactory; import org.apache.knox.gateway.topology.simple.ProviderConfigurationParser; @@ -622,6 +623,11 @@ public class DefaultTopologyService @Override public void init(GatewayConfig config, Map<String, String> options) throws ServiceLifecycleException { + String gatewayConfDir = config.getGatewayConfDir(); + if (gatewayConfDir != null) { + System.setProperty(ServiceDiscovery.CONFIG_DIR_PROPERTY, gatewayConfDir); + } + try { listeners = new HashSet<>(); topologies = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/knox/blob/21ac567d/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java ---------------------------------------------------------------------- diff --git a/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java b/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java index 01b2178..14d6c75 100644 --- a/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java +++ b/gateway-spi/src/main/java/org/apache/knox/gateway/topology/discovery/ServiceDiscovery.java @@ -26,6 +26,8 @@ import java.util.Map; */ public interface ServiceDiscovery { + String CONFIG_DIR_PROPERTY = "org.apache.knox.gateway.conf.dir"; + /** * This is the type specified in a simple descriptor to indicate which ServiceDiscovery implementation to employ. *
