Repository: ambari Updated Branches: refs/heads/trunk e44c85617 -> b3fc39395
AMBARI-10351 BE: YARN stack advisor not recommending all dependent configs (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b3fc3939 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b3fc3939 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b3fc3939 Branch: refs/heads/trunk Commit: b3fc39395d82b8b37cb2d2e5b9b8e3edec3cd0bf Parents: e44c856 Author: Dmytro Sen <[email protected]> Authored: Fri Apr 3 19:02:58 2015 +0300 Committer: Dmytro Sen <[email protected]> Committed: Fri Apr 3 19:02:58 2015 +0300 ---------------------------------------------------------------------- .../stacks/HDP/2.0.6/services/stack_advisor.py | 2 +- .../stacks/HDP/2.2/services/stack_advisor.py | 18 +- .../stacks/2.2/common/test_stack_advisor.py | 231 +++++++++++++++++++ ambari-web/pom.xml | 1 + 4 files changed, 242 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b3fc3939/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index 2405954..b2610ad 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -125,7 +125,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putYarnEnvProperty = self.putProperty(configurations, "yarn-env", services) putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer']))) putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer'])) - putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer']))) + putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])) putYarnEnvProperty('min_user_id', self.get_system_min_uid()) def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts): http://git-wip-us.apache.org/repos/asf/ambari/blob/b3fc3939/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 87748d2..3a42a3d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -169,17 +169,17 @@ class HDP22StackAdvisor(HDP21StackAdvisor): def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts): self.recommendYARNConfigurations(configurations, clusterData, services, hosts) putMapredProperty = self.putProperty(configurations, "mapred-site", services) - putMapredProperty('yarn.app.mapreduce.am.resource.mb', int(clusterData['amMemory'])) - putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(round(0.8 * clusterData['amMemory']))) + "m" + " -Dhdp.version=${hdp.version}") - putMapredProperty('mapreduce.map.memory.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) - putMapredProperty('mapreduce.reduce.memory.mb', int(clusterData['reduceMemory'])) - putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['mapMemory']))) + "m") - putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m") - putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024)) + putMapredProperty('yarn.app.mapreduce.am.resource.mb', configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) + putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(0.8 * int(configurations["mapred-site"]["properties"]["yarn.app.mapreduce.am.resource.mb"]))) + "m" + " -Dhdp.version=${hdp.version}") + putMapredProperty('mapreduce.map.memory.mb', int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])) + putMapredProperty('mapreduce.reduce.memory.mb', int(2*int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]))) + putMapredProperty('mapreduce.map.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"]))) + "m") + putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(0.8*int(configurations["mapred-site"]["properties"]["mapreduce.reduce.memory.mb"]))) + "m") + putMapredProperty('mapreduce.task.io.sort.mb', str(int(0.7*int(configurations["mapred-site"]["properties"]["mapreduce.map.memory.mb"])))) # Property Attributes putMapredPropertyAttribute = self.putPropertyAttribute(configurations, "mapred-site") - yarnMinAllocationSize = int(clusterData['ramPerContainer']) - yarnMaxAllocationSize = max(30 * int(clusterData['ramPerContainer']), configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]) + yarnMinAllocationSize = int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]) + yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"])) putMapredPropertyAttribute("mapreduce.map.memory.mb", "max", yarnMaxAllocationSize) putMapredPropertyAttribute("mapreduce.map.memory.mb", "min", yarnMinAllocationSize) putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "max", yarnMaxAllocationSize) http://git-wip-us.apache.org/repos/asf/ambari/blob/b3fc3939/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index d12c067..0cfc3b6 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -828,6 +828,237 @@ class TestHDP22StackAdvisor(TestCase): self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) + def test_recommendMapredConfigurationAttributes(self): + configurations = { + "mapred-site": { + "properties": { + "mapreduce.map.memory.mb": "1024", + "mapreduce.reduce.memory.mb": "682", + "yarn.app.mapreduce.am.command-opts": "-Xmx546m -Dhdp.version=${hdp.version}", + "mapreduce.reduce.java.opts": "-Xmx546m", + "yarn.app.mapreduce.am.resource.mb": "682", + "mapreduce.map.java.opts": "-Xmx546m", + "mapreduce.task.io.sort.mb": "273" } + }, + "yarn-site": { + "properties": { + "yarn.nodemanager.resource.memory-mb": "1280", + "yarn.scheduler.minimum-allocation-mb": "100", + "yarn.scheduler.maximum-allocation-mb": "1280", + "yarn.nodemanager.resource.cpu-vcores": "2" + }, + } + } + clusterData = { + "cpu": 4, + "containers" : 5, + "ramPerContainer": 256 + } + expected = { + "yarn-env": { + "properties": { + "min_user_id": "500" + } + }, + "mapred-site": { + "properties": { + "mapreduce.map.memory.mb": "100", + "mapreduce.reduce.memory.mb": "200", + "yarn.app.mapreduce.am.command-opts": "-Xmx80m -Dhdp.version=${hdp.version}", + "mapreduce.reduce.java.opts": "-Xmx160m", + "yarn.app.mapreduce.am.resource.mb": "100", + "mapreduce.map.java.opts": "-Xmx80m", + "mapreduce.task.io.sort.mb": "70" + }, + "property_attributes": { + 'yarn.app.mapreduce.am.resource.mb': {'max': '1280', + 'min': '100'}, + 'mapreduce.map.memory.mb': {'max': '1280', + 'min': '100'}, + 'mapreduce.reduce.memory.mb': {'max': '1280', + 'min': '100'} + } + }, + "yarn-site": { + "properties": { + "yarn.nodemanager.resource.memory-mb": "1280", + "yarn.scheduler.minimum-allocation-mb": "100", + "yarn.scheduler.maximum-allocation-mb": "1280", + "yarn.nodemanager.resource.cpu-vcores": "2" + }, + "property_attributes": { + 'yarn.nodemanager.resource.memory-mb': {'max': '1877'}, + 'yarn.nodemanager.resource.cpu-vcores': {'max': '4'}, + 'yarn.scheduler.minimum-allocation-vcores': {'max': '2'}, + 'yarn.scheduler.maximum-allocation-vcores': {'max': '2'}, + 'yarn.scheduler.minimum-allocation-mb': {'max': '1280'}, + 'yarn.scheduler.maximum-allocation-mb': {'max': '1280'} + } + } + } + services = { + "services": [ + { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2", + "StackServices" : { + "service_name" : "MAPREDUCE2", + "service_version" : "2.6.0.2.2", + "stack_name" : "HDP", + "stack_version" : "2.2" + }, + "components" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2/components/HISTORYSERVER", + "StackServiceComponents" : { + "advertise_version" : "true", + "cardinality" : "1", + "component_category" : "MASTER", + "component_name" : "HISTORYSERVER", + "custom_commands" : [ ], + "display_name" : "History Server", + "is_client" : "false", + "is_master" : "true", + "service_name" : "MAPREDUCE2", + "stack_name" : "HDP", + "stack_version" : "2.2", + "hostnames" : [ "c6402.ambari.apache.org" ] + }, + "auto_deploy" : { + "enabled" : "true", + "location" : "YARN/RESOURCEMANAGER" + }, + "dependencies" : [ { + "href" : "/api/v1/stacks/HDP/versions/2.2/services/MAPREDUCE2/components/HISTORYSERVER/dependencies/HDFS_CLIENT", + "Dependencies" : { + "component_name" : "HDFS_CLIENT", + "dependent_component_name" : "HISTORYSERVER", + "dependent_service_name" : "MAPREDUCE2", + "stack_name" : "HDP", + "stack_version" : "2.2" + } + } ] + }]}, + { + "href": "/api/v1/stacks/HDP/versions/2.2/services/YARN", + "StackServices": { + "service_name": "YARN", + "service_version": "2.6.0.2.2", + "stack_name": "HDP", + "stack_version": "2.2" + }, + "components": [ + { + "StackServiceComponents": { + "advertise_version": "false", + "cardinality": "1", + "component_category": "MASTER", + "component_name": "APP_TIMELINE_SERVER", + "display_name": "App Timeline Server", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "NODEMANAGER", + "display_name": "NodeManager", + "is_client": "false", + "is_master": "false", + "hostnames": [ + "c6403.ambari.apache.org" + ] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1-2", + "component_category": "MASTER", + "component_name": "RESOURCEMANAGER", + "display_name": "ResourceManager", + "is_client": "false", + "is_master": "true", + "hostnames": [] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1+", + "component_category": "CLIENT", + "component_name": "YARN_CLIENT", + "display_name": "YARN Client", + "is_client": "true", + "is_master": "false", + "hostnames": [] + }, + "dependencies": [] + } + ] + }, + ], + "configurations": configurations, + "changed-configurations": [ + { + "type": "yarn-site", + "name": "yarn.scheduler.minimum-allocation-mb" + }, + ] + + } + hosts = { + "items" : [ + { + "href" : "/api/v1/hosts/c6401.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6401.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6401.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + }, + { + "href" : "/api/v1/hosts/c6402.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6402.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6402.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + }, + { + "href" : "/api/v1/hosts/c6403.ambari.apache.org", + "Hosts" : { + "cpu_count" : 1, + "host_name" : "c6403.ambari.apache.org", + "os_arch" : "x86_64", + "os_type" : "centos6", + "ph_cpu_count" : 1, + "public_host_name" : "c6403.ambari.apache.org", + "rack_info" : "/default-rack", + "total_mem" : 1922680 + } + } + ] + } + self.maxDiff = None + self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, services, hosts) + self.assertEquals(configurations, expected) + def test_recommendAmsConfigurations(self): configurations = {} clusterData = {} http://git-wip-us.apache.org/repos/asf/ambari/blob/b3fc3939/ambari-web/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-web/pom.xml b/ambari-web/pom.xml index 04f9b5d..aefeff1 100644 --- a/ambari-web/pom.xml +++ b/ambari-web/pom.xml @@ -227,6 +227,7 @@ <exclude>app/assets/**</exclude> <exclude>vendor/**</exclude> <exclude>node_modules/**</exclude> + <exclude>node/**</exclude> <exclude>npm-debug.log</exclude> </excludes> </configuration>
