Repository: ambari Updated Branches: refs/heads/trunk 998246237 -> 00f454d85
AMBARI-17835 Maximum validation failure for 'yarn.scheduler.maximum-allocation-mb' after dependency change (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/00f454d8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/00f454d8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/00f454d8 Branch: refs/heads/trunk Commit: 00f454d85f9ce091dccb897d91b3b6034a53119b Parents: 9982462 Author: Dmytro Sen <[email protected]> Authored: Fri Jul 29 10:26:05 2016 +0300 Committer: Dmytro Sen <[email protected]> Committed: Fri Jul 29 10:26:05 2016 +0300 ---------------------------------------------------------------------- .../stacks/HDP/2.2/services/stack_advisor.py | 14 ++++++++------ .../stacks/2.2/common/test_stack_advisor.py | 20 +++----------------- 2 files changed, 11 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/00f454d8/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index de9ea98..a5d386e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -91,12 +91,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor): # Above is the default calculated 'maximum' values derived purely from hosts. # However, there are 'maximum' and other attributes that actually change based on the values # of other configs. We need to update those values. - - # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"]. - # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] is always 5120 during first recommendation. - # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] - # in this case (if 'changed-configurations' are empty). - if ('changed-configurations' in services.keys() and services['changed-configurations'] and "yarn-site" in services["configurations"]): + if ("yarn-site" in services["configurations"]): if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]): putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) @@ -383,9 +378,16 @@ class HDP22StackAdvisor(HDP21StackAdvisor): container_size = min(clusterData['containers'] * clusterData['ramPerContainer'], container_size, yarnMaxAllocationSize) putHiveSiteProperty("hive.tez.container.size", min(int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]), container_size)) + putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) + if "yarn-site" in services["configurations"]: + if "yarn.scheduler.minimum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]: + putHiveSitePropertyAttribute("hive.tez.container.size", "minimum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) + if "yarn.scheduler.maximum-allocation-mb" in services["configurations"]["yarn-site"]["properties"]: + putHiveSitePropertyAttribute("hive.tez.container.size", "maximum", int(services["configurations"]["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) + putHiveSiteProperty("hive.prewarm.enabled", "false") putHiveSiteProperty("hive.prewarm.numcontainers", "3") putHiveSiteProperty("hive.tez.auto.reducer.parallelism", "true") http://git-wip-us.apache.org/repos/asf/ambari/blob/00f454d8/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 467061e..29a47d9 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1162,24 +1162,10 @@ class TestHDP22StackAdvisor(TestCase): self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) - # With no 'changed-configurations', we should get updated 'maximum's from recommendations - # Else update from services. - # - # On a first stackadvisor invocation from the UI we have stack defaults in services["configurations"]. - # So services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] is always 5120 during first recommendation. - # Get a value from recommendations instead of services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] - # in this case (if 'changed-configurations' are empty). - # - # Test not empty 'changed-configurations': - services["changed-configurations"] = [ - { - "type": "yarn-env", - "name": "min_user_id" - } - ] + # Test - with no 'changed-configurations', we should get updated 'maximum's. + services.pop("changed-configurations", None) services.pop("configurations", None) - services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321', "yarn.nodemanager.resource.cpu-vcores": '9'}}, - "yarn-env": {"properties": {"min_user_id": "500"}}} + services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321', "yarn.nodemanager.resource.cpu-vcores": '9'}}} expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"] = '9' expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"] = '9' expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-mb"]["maximum"] = '4321'
