Repository: knox Updated Branches: refs/heads/master a6d2f5240 -> 98dbfe990
KNOX-1228 - Atlas HA ZooKeeper Config Discovery Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/98dbfe99 Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/98dbfe99 Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/98dbfe99 Branch: refs/heads/master Commit: 98dbfe990887c2ce032015781cc99d4ae8198cbe Parents: a6d2f52 Author: Phil Zampino <[email protected]> Authored: Thu Mar 22 13:22:13 2018 -0400 Committer: Phil Zampino <[email protected]> Committed: Wed Mar 28 09:29:04 2018 -0400 ---------------------------------------------------------------------- .../discovery/ambari/AmbariCluster.java | 38 +++++++++++- ...rvice-discovery-zk-config-mapping.properties | 14 ++++- .../discovery/ambari/AmbariClusterTest.java | 61 ++++++++++++++++++-- 3 files changed, 106 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/98dbfe99/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java index b5e4cff..9d2c92e 100644 --- a/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java +++ b/gateway-discovery-ambari/src/main/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariCluster.java @@ -19,6 +19,7 @@ package org.apache.knox.gateway.topology.discovery.ambari; import org.apache.knox.gateway.i18n.messages.MessagesFactory; import org.apache.knox.gateway.topology.discovery.ServiceDiscovery; +import java.io.File; import java.io.FileInputStream; import java.io.InputStream; import java.util.ArrayList; @@ -33,6 +34,8 @@ class AmbariCluster implements ServiceDiscovery.Cluster { private static final String ZK_CONFIG_MAPPING_FILE = "ambari-service-discovery-zk-config-mapping.properties"; + private static final String ZK_CONFIG_MAPPING_OVERRIDE_FILE = "ambari-discovery-zk-config.properties"; + static final String ZK_CONFIG_MAPPING_SYSTEM_PROPERTY = "org.apache.knox.gateway.topology.discovery.ambari.zk.mapping"; @@ -48,7 +51,22 @@ class AmbariCluster implements ServiceDiscovery.Cluster { } // Attempt to apply overriding or additional mappings from external source - String overridesPath = System.getProperty(ZK_CONFIG_MAPPING_SYSTEM_PROPERTY); + String overridesPath = null; + + // First, check for the well-known overrides config file + String gatewayConfDir = System.getProperty(ServiceDiscovery.CONFIG_DIR_PROPERTY); + if (gatewayConfDir != null) { + File overridesFile = new File(gatewayConfDir, ZK_CONFIG_MAPPING_OVERRIDE_FILE); + if (overridesFile.exists()) { + overridesPath = overridesFile.getAbsolutePath(); + } + } + + // If no file in the config dir, check for the system property reference + if (overridesPath == null) { + overridesPath = System.getProperty(ZK_CONFIG_MAPPING_SYSTEM_PROPERTY); + } + if (overridesPath != null) { Properties overrides = new Properties(); try (InputStream in = new FileInputStream(overridesPath)) { @@ -150,6 +168,24 @@ class AmbariCluster implements ServiceDiscovery.Cluster { String namespaceProp = zooKeeperHAConfigMappings.getProperty(serviceName + ".namespace"); Map<String, String> scProps = sc.getProperties(); if (scProps != null) { + + if (ensembleProp != null) { + // If there are multiple ensemble properties specified, then iteratively check for the first + // valid value, and use that one. + String[] ensembleProps = ensembleProp.split(","); + if (ensembleProps.length > 1) { + for (String prop : ensembleProps) { + if (!prop.isEmpty()) { + String value = scProps.get(prop); + if (value != null) { + ensembleProp = prop; + break; + } + } + } + } + } + result = new ZooKeeperConfiguration(enabledProp != null ? scProps.get(enabledProp) : null, ensembleProp != null ? scProps.get(ensembleProp) : null, http://git-wip-us.apache.org/repos/asf/knox/blob/98dbfe99/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties index 0a5faae..752382e 100644 --- a/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties +++ b/gateway-discovery-ambari/src/main/resources/ambari-service-discovery-zk-config-mapping.properties @@ -48,4 +48,16 @@ YARN.ensemble=yarn.resourcemanager.zk-address # WEBHCAT WEBHCAT.config=WEBHCAT:webhcat-site -WEBHCAT.ensemble=templeton.zookeeper.hosts \ No newline at end of file +WEBHCAT.ensemble=templeton.zookeeper.hosts + +# ATLAS +ATLAS.config=ATLAS:application-properties +ATLAS.enabled=atlas.server.ha.enabled +ATLAS.ensemble=atlas.server.ha.zookeeper.connect,atlas.kafka.zookeeper.connect +ATLAS.namespace=atlas.server.ha.zookeeper.zkroot + +# ATLAS-API +ATLAS-API.config=ATLAS:application-properties +ATLAS-API.enabled=atlas.server.ha.enabled +ATLAS-API.ensemble=atlas.server.ha.zookeeper.connect,atlas.kafka.zookeeper.connect +ATLAS-API.namespace=atlas.server.ha.zookeeper.zkroot http://git-wip-us.apache.org/repos/asf/knox/blob/98dbfe99/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java index abec9d9..c51da6a 100644 --- a/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java +++ b/gateway-discovery-ambari/src/test/java/org/apache/knox/gateway/topology/discovery/ambari/AmbariClusterTest.java @@ -16,17 +16,12 @@ */ package org.apache.knox.gateway.topology.discovery.ambari; -import org.apache.commons.io.FileUtils; import org.apache.knox.gateway.topology.discovery.ServiceDiscovery; -import org.apache.knox.test.TestUtils; import org.easymock.EasyMock; import org.junit.Test; -import java.io.File; -import java.io.FileOutputStream; import java.util.HashMap; import java.util.Map; -import java.util.Properties; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -143,6 +138,62 @@ public class AmbariClusterTest { } + /** + * The Atlas ZooKeeper ensemble determination is based on multiple properties with a prioritized search order. + * This test verifies that the default property is used when the primary property value is undefined. + */ + @Test + public void testAtlasZooKeeperEnsemblePropsConfigurationSecondary() throws Exception { + + final boolean isEnabled = true; + final String ensemble = "host1:2181,host2:2181,host3:2181"; + //final String ensemblePropPrimary = "atlas.server.ha.zookeeper.connect"; + final String ensemblePropSecondary = "atlas.kafka.zookeeper.connect"; + final String namespace = "/apache_atlas"; + + Map<String, String> serviceConfigProps = new HashMap<>(); + // Since the primary property is undefined, the ZooKeeper config should query the value of the secondary property + serviceConfigProps.put(ensemblePropSecondary, ensemble); + serviceConfigProps.put("atlas.server.ha.zookeeper.zkroot", namespace); + serviceConfigProps.put("atlas.server.ha.enabled", "true"); + + AmbariCluster.ZooKeeperConfig config = + getZooKeeperConfiguration("ATLAS", "ATLAS", "application-properties", serviceConfigProps); + assertNotNull(config); + assertEquals(isEnabled, config.isEnabled()); + assertEquals(ensemble, config.getEnsemble()); + assertEquals(namespace, config.getNamespace()); + } + + + /** + * The Atlas ZooKeeper ensemble determination is based on multiple properties with a prioritized search order. + * This test verifies that the primary property value is used when it's defined. + */ + @Test + public void testAtlasZooKeeperEnsemblePropsConfigurationPrimary() throws Exception { + + final boolean isEnabled = true; + final String ensemble = "host1:2181,host2:2181,host3:2181"; + final String ensemblePropPrimary = "atlas.server.ha.zookeeper.connect"; + final String ensemblePropSecondary = "atlas.kafka.zookeeper.connect"; + final String namespace = "/apache_atlas"; + + Map<String, String> serviceConfigProps = new HashMap<>(); + serviceConfigProps.put(ensemblePropSecondary, "invalid"); + serviceConfigProps.put(ensemblePropPrimary, ensemble); // Since the primary property is defined, its value should be used + serviceConfigProps.put("atlas.server.ha.zookeeper.zkroot", namespace); + serviceConfigProps.put("atlas.server.ha.enabled", "true"); + + AmbariCluster.ZooKeeperConfig config = + getZooKeeperConfiguration("ATLAS", "ATLAS", "application-properties", serviceConfigProps); + assertNotNull(config); + assertEquals(isEnabled, config.isEnabled()); + assertEquals(ensemble, config.getEnsemble()); + assertEquals(namespace, config.getNamespace()); + } + + private ServiceDiscovery.Cluster.ZooKeeperConfig getZooKeeperConfiguration(final String serviceName, final String configType,
