BIGTOP-2663: puppet hadoop module: Consolidate memory resource settings
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/f7a92168 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/f7a92168 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/f7a92168 Branch: refs/heads/master Commit: f7a92168debe9c3095b08b2fdf16176060781c47 Parents: 3082e73 Author: Olaf Flebbe <[email protected]> Authored: Fri Jan 6 22:52:03 2017 +0100 Committer: Olaf Flebbe <[email protected]> Committed: Sat Jan 7 17:44:50 2017 +0100 ---------------------------------------------------------------------- .../puppet/modules/hadoop/manifests/init.pp | 4 +++ .../modules/hadoop/templates/mapred-site.xml | 31 +++++++++++++++++--- .../modules/hadoop/templates/yarn-site.xml | 8 +++++ 3 files changed, 39 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/f7a92168/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp index a8a8152..30a68e2 100644 --- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp @@ -382,6 +382,10 @@ class hadoop ($hadoop_security_authentication = "simple", $mapreduce_output_fileoutputformat_compress_type = "BLOCK", # "RECORD" default $mapreduce_map_output_compress = undef, $mapreduce_job_reduce_slowstart_completedmaps = undef, + $mapreduce_map_memory_mb = undef, + $mapreduce_reduce_memory_mb = undef, + $mapreduce_map_java_opts = "-Xmx1024m", + $mapreduce_reduce_java_opts = "-Xmx1024m", $hadoop_security_authentication = $hadoop::hadoop_security_authentication, $kerberos_realm = $hadoop::kerberos_realm, ) inherits hadoop { http://git-wip-us.apache.org/repos/asf/bigtop/blob/f7a92168/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml index ea49926..a25f66b 100644 --- a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml +++ b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml @@ -111,20 +111,44 @@ <name>yarn.app.mapreduce.am.staging-dir</name> <value><%= @yarn_app_mapreduce_am_staging_dir %></value> </property> +<% end -%> + +<!-- Memory settings --> +<% if @mapreduce_map_memory_mb -%> + <property> + <name>mapreduce.map.memory.mb</name> + <value><%= @mapreduce_map_memory_mb %></value> + </property> +<% end -%> + +<% if @mapreduce_reduce_memory_mb -%> + <property> + <name>mapreduce.reduce.memory.mb</name> + <value><%= @mapreduce_reduce_memory_mb %></value> + </property> +<% end -%> +<% if @mapreduce_map_java_opts -%> + <property> + <name>mapreduce.map.java.opts</name> + <value><%= @mapreduce_map_java_opts %></value> + </property> <% end -%> + +<% if @mapreduce_reduce_java_opts -%> <property> - <name>mapred.child.java.opts</name> - <value>-Xmx1024m</value> + <name>mapreduce.reduce.java.opts</name> + <value><%= @mapreduce_reduce_java_opts %></value> </property> +<% end -%> <% if @mapreduce_task_io_sort_mb -%> <property> <name>mapreduce.task.io.sort.mb</name> <value><%= @mapreduce_task_io_sort_mb %></value> </property> - <% end -%> + <% if @mapreduce_task_io_sort_factor -%> <property> <name>mapreduce.task.io.sort.factor</name> @@ -189,6 +213,5 @@ <name>mapreduce.job.reduce.slowstart.completedmaps</name> <value><%= @mapreduce_job_reduce_slowstart_completedmaps %></value> </property> - <% end -%> </configuration> http://git-wip-us.apache.org/repos/asf/bigtop/blob/f7a92168/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml index 56c6015..e9e6a05 100644 --- a/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml +++ b/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml @@ -234,4 +234,12 @@ <value><%= @yarn_resourcemanager_scheduler_class %></value> </property> <% end -%> + +<!-- The defaut setting (2.1) is silly. The virtual memory is not + a limiting factor on 64Bit systems, at least not a limiting + resource, so make it large, very large. --> + <property> + <name>yarn.nodemanager.vmem-pmem-ratio</name> + <value>100</value> + </property> </configuration>
