Repository: ambari Updated Branches: refs/heads/branch-2.4 c411be395 -> 8cdd83594
AMBARI-17802. Blueprint deployment configures "org.apache.atlas.hive.hook.HiveHook" twice for "hive.exec.post.hooks" config (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8cdd8359 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8cdd8359 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8cdd8359 Branch: refs/heads/branch-2.4 Commit: 8cdd83594ba2880d745b472ba6667a9805afd53e Parents: c411be3 Author: Alejandro Fernandez <[email protected]> Authored: Tue Jul 19 19:00:53 2016 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Wed Jul 20 11:38:28 2016 -0700 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 32 +++- .../stacks/HDP/2.3/services/stack_advisor.py | 39 ++--- .../BlueprintConfigurationProcessorTest.java | 147 ++++++++++++------- 3 files changed, 142 insertions(+), 76 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8cdd8359/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index a295bf7..23c9edc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -2440,16 +2440,34 @@ public class BlueprintConfigurationProcessor { String origValue, Map<String, Map<String, String>> properties, ClusterTopology topology) { + String atlasHookClass = "org.apache.atlas.hive.hook.HiveHook"; + String[] hiveHooks = origValue.split(","); - if (topology.getBlueprint().getServices().contains("ATLAS")) { - String hiveHookClass = "org.apache.atlas.hive.hook.HiveHook"; - if (origValue == null || origValue.isEmpty()) { - return hiveHookClass; - } else { - return String.format("%s,%s", origValue, hiveHookClass); + List<String> hiveHooksClean = new ArrayList<String>(); + for(String hiveHook : hiveHooks) { + if (!StringUtils.isBlank(hiveHook.trim())) { + hiveHooksClean.add(hiveHook.trim()); + } + } + + boolean isAtlasInCluster = topology.getBlueprint().getServices().contains("ATLAS"); + + // Append atlas hook if not already present. + if (isAtlasInCluster) { + if (!hiveHooksClean.contains(atlasHookClass)) { + hiveHooksClean.add(atlasHookClass); } } else { - return origValue; + // Remove the atlas hook since Atlas service is not present. + while (hiveHooksClean.contains(atlasHookClass)) { + hiveHooksClean.remove(atlasHookClass); + } + } + + if (!hiveHooksClean.isEmpty()) { + return StringUtils.join(hiveHooksClean, ","); + } else { + return " "; } } }); http://git-wip-us.apache.org/repos/asf/ambari/blob/8cdd8359/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index e1bedb7..19753e5 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -232,34 +232,35 @@ class HDP23StackAdvisor(HDP22StackAdvisor): else: putHiveSitePropertyAttribute('datanucleus.rdbms.datastoreAdapterClassName', 'delete', 'true') - # atlas + # Atlas hooks_property = "hive.exec.post.hooks" + atlas_hook_class = "org.apache.atlas.hive.hook.HiveHook" if hooks_property in configurations["hive-site"]["properties"]: hooks_value = configurations["hive-site"]["properties"][hooks_property] else: - hooks_value = " " + hooks_value = "" - include_atlas = "ATLAS" in servicesList - atlas_hook_class = "org.apache.atlas.hive.hook.HiveHook" - if include_atlas and atlas_hook_class not in hooks_value: - if hooks_value == " ": - hooks_value = atlas_hook_class - else: - hooks_value = hooks_value + "," + atlas_hook_class - if not include_atlas and atlas_hook_class in hooks_value: - hooks_classes = [] - for hook_class in hooks_value.split(","): - if hook_class != atlas_hook_class and hook_class != " ": - hooks_classes.append(hook_class) - if hooks_classes: - hooks_value = ",".join(hooks_classes) - else: - hooks_value = " " + + hive_hooks = [x.strip() for x in hooks_value.split(",")] + hive_hooks = [x for x in hive_hooks if x != ""] + is_atlas_present_in_cluster = "ATLAS" in servicesList + + if is_atlas_present_in_cluster: + # Append atlas hook if not already present. + is_atlas_hook_in_config = atlas_hook_class in hive_hooks + if not is_atlas_hook_in_config: + hive_hooks.append(atlas_hook_class) + else: + # Remove the atlas hook since Atlas service is not present. + hive_hooks = [x for x in hive_hooks if x != atlas_hook_class] + + # Convert hive_hooks back to a csv, unless there are 0 elements, which should be " " + hooks_value = " " if len(hive_hooks) == 0 else ",".join(hive_hooks) putHiveSiteProperty(hooks_property, hooks_value) # This is no longer used in HDP 2.5, but still needed in HDP 2.3 and 2.4 atlas_server_host_info = self.getHostWithComponent("ATLAS", "ATLAS_SERVER", services, hosts) - if include_atlas and atlas_server_host_info: + if is_atlas_present_in_cluster and atlas_server_host_info: atlas_rest_host = atlas_server_host_info['Hosts']['host_name'] scheme = "http" metadata_port = "21000" http://git-wip-us.apache.org/repos/asf/ambari/blob/8cdd8359/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 268bc29..7aa4a4e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -5828,29 +5828,75 @@ public class BlueprintConfigurationProcessorTest { @Test public void testAtlasHiveProperties() throws Exception { + Map<String, Map<String, String>> properties = getAtlasHivePropertiesForTestCase(); + validateAtlasHivePropertiesForTestCase(properties); + } + + /** + * If the Hive Exec Hooks property doesn't contain the Atlas Hook, then add it. + * @throws Exception + */ + @Test + public void testAtlasHivePropertiesWithHiveHookSpace() throws Exception { + Map<String, Map<String, String>> properties = getAtlasHivePropertiesForTestCase(); + + Map<String, String> hiveProperties = properties.get("hive-site"); + hiveProperties.put("hive.exec.post.hooks", " "); + properties.put("hive-site", hiveProperties); + validateAtlasHivePropertiesForTestCase(properties); + } + + /*** + * If the Atlas Hook already exists, don't append it. + * @throws Exception + */ + @Test + public void testAtlasHivePropertiesWithAtlasHookAlreadyExist() throws Exception { + Map<String, Map<String, String>> properties = getAtlasHivePropertiesForTestCase(); + + Map<String, String> hiveProperties = properties.get("hive-site"); + hiveProperties.put("hive.exec.post.hooks", "org.apache.atlas.hive.hook.HiveHook"); + properties.put("hive-site", hiveProperties); + validateAtlasHivePropertiesForTestCase(properties); + } + + /** + * Generate sample collection of properties for some of the test cases. + * @return Map of sample properties + */ + private Map<String, Map<String, String>> getAtlasHivePropertiesForTestCase() { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); + Map<String, String> atlasProperties = new HashMap<String, String>(); - properties.put("application-properties", atlasProperties); atlasProperties.put("atlas.enableTLS", "false"); atlasProperties.put("atlas.server.bind.address", "localhost"); atlasProperties.put("atlas.server.http.port", "21000"); - Map<String, String> atlasEnv = new HashMap<String, String>(); + properties.put("application-properties", atlasProperties); + Map<String, String> atlasEnv = new HashMap<String, String>(); properties.put("atlas-env", atlasEnv); + Map<String, String> hiveProperties = new HashMap<String, String>(); hiveProperties.put("hive.exec.post.hooks", ""); hiveProperties.put("atlas.cluster.name", "primary"); hiveProperties.put("atlas.rest.address", "http://localhost:21000";); properties.put("hive-site", hiveProperties); + return properties; + } + /** + * For several test cases, validate that org.apache.atlas.hive.hook.HiveHook has the correct value. + * @param properties Map of properties to validate + * @throws Exception + */ + private void validateAtlasHivePropertiesForTestCase(Map<String, Map<String, String>> properties) throws Exception { Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); Configuration parentClusterConfig = new Configuration(parentProperties, Collections.<String, Map<String, Map<String, String>>>emptyMap()); Configuration clusterConfig = new Configuration(properties, Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); - Collection<String> hgComponents1 = new HashSet<String>(); hgComponents1.add("ATLAS_SERVER"); hgComponents1.add("HIVE_SERVER"); @@ -5858,8 +5904,8 @@ public class BlueprintConfigurationProcessorTest { Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); - BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); + ClusterTopology topology1 = createClusterTopology(bp, clusterConfig, hostGroups); + BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology1); configProcessor.doUpdateForClusterCreate(); @@ -5870,6 +5916,52 @@ public class BlueprintConfigurationProcessorTest { } @Test + public void testAtlasHivePropertiesWithHTTPS() throws Exception { + Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); + + Map<String, String> atlasProperties = new HashMap<String, String>(); + properties.put("application-properties", atlasProperties); + // use https + atlasProperties.put("atlas.enableTLS", "true"); + atlasProperties.put("atlas.server.bind.address", "localhost"); + atlasProperties.put("atlas.server.https.port", "99999"); + Map<String, String> atlasEnv = new HashMap<String, String>(); + + properties.put("atlas-env", atlasEnv); + Map<String, String> hiveProperties = new HashMap<String, String>(); + // default hook registered + hiveProperties.put("hive.exec.post.hooks", "foo"); + // user specified cluster name + hiveProperties.put("atlas.cluster.name", "userSpecified"); + hiveProperties.put("atlas.rest.address", "http://localhost:21000";); + properties.put("hive-site", hiveProperties); + + + Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); + Configuration parentClusterConfig = new Configuration(parentProperties, + Collections.<String, Map<String, Map<String, String>>>emptyMap()); + Configuration clusterConfig = new Configuration(properties, + Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); + + + Collection<String> hgComponents1 = new HashSet<String>(); + hgComponents1.add("ATLAS_SERVER"); + hgComponents1.add("HIVE_SERVER"); + TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); + + Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); + + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); + + configProcessor.doUpdateForClusterCreate(); + + assertEquals("foo,org.apache.atlas.hive.hook.HiveHook", clusterConfig.getPropertyValue("hive-site", "hive.exec.post.hooks")); + assertEquals("userSpecified", clusterConfig.getPropertyValue("hive-site", "atlas.cluster.name")); + assertEquals("https://host1:99999";, clusterConfig.getPropertyValue("hive-site", "atlas.rest.address")); + } + + @Test public void testStormAmsPropertiesDefault() throws Exception { Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); @@ -5996,51 +6088,6 @@ public class BlueprintConfigurationProcessorTest { } @Test - public void testAtlasHiveProperties2() throws Exception { - Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); - Map<String, String> atlasProperties = new HashMap<String, String>(); - properties.put("application-properties", atlasProperties); - // use https - atlasProperties.put("atlas.enableTLS", "true"); - atlasProperties.put("atlas.server.bind.address", "localhost"); - atlasProperties.put("atlas.server.https.port", "99999"); - Map<String, String> atlasEnv = new HashMap<String, String>(); - - properties.put("atlas-env", atlasEnv); - Map<String, String> hiveProperties = new HashMap<String, String>(); - // default hook registered - hiveProperties.put("hive.exec.post.hooks", "foo"); - // user specified cluster name - hiveProperties.put("atlas.cluster.name", "userSpecified"); - hiveProperties.put("atlas.rest.address", "http://localhost:21000";); - properties.put("hive-site", hiveProperties); - - - Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); - Configuration parentClusterConfig = new Configuration(parentProperties, - Collections.<String, Map<String, Map<String, String>>>emptyMap()); - Configuration clusterConfig = new Configuration(properties, - Collections.<String, Map<String, Map<String, String>>>emptyMap(), parentClusterConfig); - - - Collection<String> hgComponents1 = new HashSet<String>(); - hgComponents1.add("ATLAS_SERVER"); - hgComponents1.add("HIVE_SERVER"); - TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); - - Collection<TestHostGroup> hostGroups = Collections.singletonList(group1); - - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); - BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); - - configProcessor.doUpdateForClusterCreate(); - - assertEquals("foo,org.apache.atlas.hive.hook.HiveHook", clusterConfig.getPropertyValue("hive-site", "hive.exec.post.hooks")); - assertEquals("userSpecified", clusterConfig.getPropertyValue("hive-site", "atlas.cluster.name")); - assertEquals("https://host1:99999";, clusterConfig.getPropertyValue("hive-site", "atlas.rest.address")); - } - - @Test public void testRecommendConfiguration_applyStackDefaultsOnly() throws Exception { // GIVEN final String expectedHostName = "c6401.apache.ambari.org";
