Repository: ambari Updated Branches: refs/heads/trunk ebabac069 -> 9bcd3f378
AMBARI-19404 Warning by default by stack advisor (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9bcd3f37 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9bcd3f37 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9bcd3f37 Branch: refs/heads/trunk Commit: 9bcd3f3784d985da6b5515aa6ec7ece37a3329b0 Parents: ebabac0 Author: Dmytro Sen <[email protected]> Authored: Fri Jan 6 19:56:27 2017 +0200 Committer: Dmytro Sen <[email protected]> Committed: Fri Jan 6 19:56:27 2017 +0200 ---------------------------------------------------------------------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 10 ++-- .../stacks/HDP/2.2/services/stack_advisor.py | 11 ---- .../stacks/2.0.6/common/test_stack_advisor.py | 54 ++++++++++++++++++++ .../stacks/2.2/common/test_stack_advisor.py | 27 ++++++++-- 4 files changed, 83 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9bcd3f37/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index b85ddf8..62f4e3e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -456,12 +456,14 @@ class HDP206StackAdvisor(DefaultStackAdvisor): self.updateMountProperties("hdfs-site", hdfs_mount_properties, configurations, services, hosts) - if hdfsSiteProperties and "dfs.datanode.data.dir" in hdfsSiteProperties and\ - hdfsSiteProperties["dfs.datanode.data.dir"] is not None: + if configurations and "hdfs-site" in configurations and \ + "dfs.datanode.data.dir" in configurations["hdfs-site"]["properties"] and \ + configurations["hdfs-site"]["properties"]["dfs.datanode.data.dir"] is not None: + dataDirs = configurations["hdfs-site"]["properties"]["dfs.datanode.data.dir"].split(",") + elif hdfsSiteProperties and "dfs.datanode.data.dir" in hdfsSiteProperties and \ + hdfsSiteProperties["dfs.datanode.data.dir"] is not None: dataDirs = hdfsSiteProperties["dfs.datanode.data.dir"].split(",") - else: - dataDirs = configurations["hdfs-site"]["properties"]["dfs.datanode.data.dir"].split(",") # dfs.datanode.du.reserved should be set to 10-15% of volume size # For each host selects maximum size of the volume. Then gets minimum for all hosts. http://git-wip-us.apache.org/repos/asf/ambari/blob/9bcd3f37/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index b61b0ab..63e4b20 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -100,17 +100,6 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) - # Above is the default calculated 'maximum' values derived purely from hosts. - # However, there are 'maximum' and other attributes that actually change based on the values - # of other configs. We need to update those values. - if ("yarn-site" in services["configurations"]): - if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]): - putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) - putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]) - if ("yarn.nodemanager.resource.cpu-vcores" in services["configurations"]["yarn-site"]["properties"]): - putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-vcores', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"]) - putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-vcores', 'maximum', services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"]) - kerberos_authentication_enabled = self.isSecurityEnabled(services) if kerberos_authentication_enabled: putYarnProperty('yarn.nodemanager.container-executor.class', http://git-wip-us.apache.org/repos/asf/ambari/blob/9bcd3f37/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py index ca697c0..ab7d53b 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py @@ -1513,6 +1513,60 @@ class TestHDP206StackAdvisor(TestCase): self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services3, hosts) self.assertEquals(configurations, expected) + hosts = { + "items": [ + { + "href": "/api/v1/hosts/host1", + "Hosts": { + "cpu_count": 1, + "host_name": "c6401.ambari.apache.org", + "os_arch": "x86_64", + "os_type": "centos6", + "ph_cpu_count": 1, + "public_host_name": "public.c6401.ambari.apache.org", + "rack_info": "/default-rack", + "total_mem": 2097152, + "disk_info": [{ + "available" : "21039512", + "device" : "/dev/vda1", + "used" : "3316924", + "percent" : "14%", + "size" : "25666616", + "type" : "ext4", + "mountpoint" : "/" + }, + { + "available" : "244732200", + "device" : "/dev/vdb", + "used" : "60508", + "percent" : "1%", + "size" : "257899908", + "type" : "ext4", + "mountpoint" : "/grid/0" + }] + } + } + ]} + + expected["hdfs-site"] = { + 'properties': { + 'dfs.datanode.du.reserved': '33011188224', + 'dfs.internal.nameservices': 'mycluster', + 'dfs.ha.namenodes.mycluster': 'nn1,nn2', + 'dfs.datanode.data.dir': '/hadoop/hdfs/data,/grid/0/hadoop/hdfs/data', + 'dfs.namenode.name.dir': '/hadoop/hdfs/namenode,/grid/0/hadoop/hdfs/namenode', + 'dfs.namenode.checkpoint.dir': '/hadoop/hdfs/namesecondary', + }, + 'property_attributes': { + 'dfs.namenode.rpc-address': { + 'delete': 'true' + } + } + } + self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services3, hosts) + self.assertEquals(configurations, expected) + + def test_recommendAmsConfigurations(self): configurations = { http://git-wip-us.apache.org/repos/asf/ambari/blob/9bcd3f37/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 1c9238e..2a61b34 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1168,17 +1168,36 @@ class TestHDP22StackAdvisor(TestCase): self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) - # Test - with no 'changed-configurations', we should get updated 'maximum's. services.pop("changed-configurations", None) + services["changed-configurations"] = [{ + "type": "yarn-site", + "name": "yarn.nodemanager.resource.memory-mb", + "old_value": "1280" + }] services.pop("configurations", None) - services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321', "yarn.nodemanager.resource.cpu-vcores": '9'}}} - expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"] = '9' - expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"] = '9' + services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.memory-mb": '4321'}}} + + expected["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"] = '4321' + expected["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"] = '4321' expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-mb"]["maximum"] = '4321' expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-mb"]["maximum"] = '4321' self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) + services["changed-configurations"].append({ + "type": "yarn-site", + "name": "yarn.nodemanager.resource.cpu-vcores", + "old_value": "7" + }) + services.pop("configurations", None) + services["configurations"] = {"yarn-site": {"properties": {"yarn.nodemanager.resource.cpu-vcores": '9', "yarn.nodemanager.resource.memory-mb": '4321'}}} + expected["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"] = '9' + expected["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-vcores"] = '9' + expected["yarn-site"]["property_attributes"]["yarn.scheduler.maximum-allocation-vcores"]["maximum"] = '9' + expected["yarn-site"]["property_attributes"]["yarn.scheduler.minimum-allocation-vcores"]["maximum"] = '9' + self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) + def test_multipleDependsOn(self): configurations = { "yarn-env": {
