Repository: ambari Updated Branches: refs/heads/trunk 4acca9415 -> 59364bc87
AMBARI-16919. Handle the scenario when 'capacity-scheduler' is passed-in as dictionary to Stack Advisor and services['configurations'][capacity-scheduler][properties][capacity-scheduler] is null. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/59364bc8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/59364bc8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/59364bc8 Branch: refs/heads/trunk Commit: 59364bc87d0ad3c6fa16918329d40230b34646c8 Parents: 08d70f5 Author: Swapan Shridhar <[email protected]> Authored: Thu May 26 18:47:21 2016 -0700 Committer: Swapan Shridhar <[email protected]> Committed: Thu May 26 19:58:44 2016 -0700 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 4 ++- .../stacks/HDP/2.5/services/stack_advisor.py | 28 +++++++++++++++----- .../stacks/2.5/common/test_stack_advisor.py | 7 +++-- 3 files changed, 30 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/59364bc8/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 4e404bc..a0af813 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -448,6 +448,8 @@ public class BlueprintConfigurationProcessor { private void doRecommendConfigurations(Configuration configuration, Set<String> configTypesUpdated) { ConfigRecommendationStrategy configRecommendationStrategy = clusterTopology.getConfigRecommendationStrategy(); Map<String, AdvisedConfiguration> advisedConfigurations = clusterTopology.getAdvisedConfigurations(); + LOG.info("Config recommendation strategy being used is {})", configRecommendationStrategy); + if (ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY.equals(configRecommendationStrategy)) { LOG.info("Filter out recommended configurations. Keep only the stack defaults."); doFilterStackDefaults(advisedConfigurations); @@ -465,7 +467,7 @@ public class BlueprintConfigurationProcessor { } } } else { - LOG.info("No any recommended configuration applied. (strategy: {})", ConfigRecommendationStrategy.NEVER_APPLY); + LOG.info("No recommended configurations are applied. (strategy: {})", ConfigRecommendationStrategy.NEVER_APPLY); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/59364bc8/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 44407ca..8c5351f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -1025,17 +1025,33 @@ class HDP25StackAdvisor(HDP24StackAdvisor): received_as_key_value_pair = True if "capacity-scheduler" in services['configurations']: if "capacity-scheduler" in services['configurations']["capacity-scheduler"]["properties"]: - properties = str(services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"]).split('\n') - if properties: - if properties[0] != 'null': + cap_sched_props_as_str = services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"] + if cap_sched_props_as_str: + cap_sched_props_as_str = str(cap_sched_props_as_str).split('\n') + if len(cap_sched_props_as_str) > 0 and cap_sched_props_as_str[0] != 'null': # Received confgs as one "\n" separated string - for property in properties: + for property in cap_sched_props_as_str: key, sep, value = property.partition("=") capacity_scheduler_properties[key] = value + Logger.info("'capacity-scheduler' configs is passed-in as a single '\\n' separated string. " + "count(services['configurations']['capacity-scheduler']['properties']['capacity-scheduler']) = " + "{0}".format(len(capacity_scheduler_properties))) received_as_key_value_pair = False else: - # Received configs as a dictionary (Generally on 1st invocation). - capacity_scheduler_properties = services['configurations']["capacity-scheduler"]["properties"] + Logger.info("Passed-in services['configurations']['capacity-scheduler']['properties']['capacity-scheduler'] is 'null'.") + else: + Logger.info("'capacity-schdeuler' configs not passed-in as single '\\n' string in " + "services['configurations']['capacity-scheduler']['properties']['capacity-scheduler'].") + if not capacity_scheduler_properties: + # Received configs as a dictionary (Generally on 1st invocation). + capacity_scheduler_properties = services['configurations']["capacity-scheduler"]["properties"] + Logger.info("'capacity-scheduler' configs is passed-in as a dictionary. " + "count(services['configurations']['capacity-scheduler']['properties']) = {0}".format(len(capacity_scheduler_properties))) + else: + Logger.error("Couldn't retrieve 'capacity-scheduler' from services.") + + Logger.info("Retrieved 'capacity-scheduler' received as dictionary : '{0}'. configs : {1}"\ + .format(received_as_key_value_pair, capacity_scheduler_properties.items())) return capacity_scheduler_properties, received_as_key_value_pair def recommendRangerKMSConfigurations(self, configurations, clusterData, services, hosts): http://git-wip-us.apache.org/repos/asf/ambari/blob/59364bc8/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 01064a6..0066e1d 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -636,6 +636,7 @@ class TestHDP25StackAdvisor(TestCase): # Test 2 : (1). Only default queue exists in capacity-scheduler and capacity-scheduler is passed-in as a dictionary, + # and services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"] is set to value "null" # (2). enable_hive_interactive' is 'On' and 'llap_queue_capacity is 0. def test_recommendYARNConfigurations_create_llap_queue_1(self): @@ -913,6 +914,7 @@ class TestHDP25StackAdvisor(TestCase): # Test 4: (1). Only default queue exists in capacity-scheduler and capacity-scheduler is passed-in as a dictionary + # and services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"] is null # (2). enable_hive_interactive' is 'On' and 'llap_queue_capacity is 40. def test_recommendYARNConfigurations_create_llap_queue_2(self): services = { @@ -977,7 +979,7 @@ class TestHDP25StackAdvisor(TestCase): "configurations": { "capacity-scheduler" : { "properties" : { - "capacity-scheduler" : "null", + "capacity-scheduler" : None, "yarn.scheduler.capacity.root.accessible-node-labels" : "*", "yarn.scheduler.capacity.maximum-am-resource-percent" : "1", "yarn.scheduler.capacity.root.acl_administer_queue" : "*", @@ -4606,7 +4608,8 @@ class TestHDP25StackAdvisor(TestCase): # Test 28: (1). 'default' and 'llap' (State : RUNNING) queue exists at root level in capacity-scheduler, and - # 'capacity-scheduler' configs are passed-in as dictionary and + # 'capacity-scheduler' configs are passed-in as dictionary and + # services['configurations']["capacity-scheduler"]["properties"]["capacity-scheduler"] is set to value "null" and # (2). enable_hive_interactive' is 'on' and (3). configuration change detected for 'hive.server2.tez.sessions.per.default.queue' # Expected : Configurations values recommended for llap related configs. def test_recommendYARNConfigurations_five_node_manager_llap_configs_updated_3(self):
