Ottomata has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/316372

Change subject: Override nofiles ulimit for nodemanager processes
......................................................................

Override nofiles ulimit for nodemanager processes

Bug: T148206
Change-Id: I0fcfde5acb7358dc7435fb60a494a87f82bfd879
---
M manifests/hadoop.pp
M manifests/hadoop/defaults.pp
M manifests/hadoop/nodemanager.pp
A templates/hadoop/hadoop-yarn-nodemanager.default.erb
4 files changed, 21 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet/cdh 
refs/changes/72/316372/1

diff --git a/manifests/hadoop.pp b/manifests/hadoop.pp
index 079d687..86bfc7e 100644
--- a/manifests/hadoop.pp
+++ b/manifests/hadoop.pp
@@ -77,6 +77,7 @@
 #                                               Default: RECORD
 #   $yarn_nodemanager_resource_memory_mb
 #   $yarn_nodemanager_resource_cpu_vcores     - Default: max($::processorcount 
- 1, 1)
+#   $yarn_nodemanager_nofiles_ulimit          - Default: 65536
 #   $yarn_scheduler_minimum_allocation_mb     - The minimum allocation for 
every container request at the RM,
 #                                               in MBs. Memory requests lower 
than this won't take effect, and
 #                                               the specified value will get 
allocated at minimum.
@@ -167,6 +168,7 @@
     $mapreduce_output_compression_type           = 
$::cdh::hadoop::defaults::mapreduce_output_compression_type,
     $yarn_nodemanager_resource_memory_mb         = 
$::cdh::hadoop::defaults::yarn_nodemanager_resource_memory_mb,
     $yarn_nodemanager_resource_cpu_vcores        = 
$::cdh::hadoop::defaults::yarn_nodemanager_resource_cpu_vcores,
+    $yarn_nodemanager_nofiles_ulimit             = 
$::cdh::hadoop::defaults::yarn_nodemanager_nofiles_ulimit,
     $yarn_log_aggregation_retain_seconds         = 
$::cdh::hadoop::defaults::yarn_log_aggregation_retain_seconds,
     $yarn_log_aggregation_retain_check_interval_seconds = 
$::cdh::hadoop::defaults::yarn_log_aggregation_retain_check_interval_seconds,
     $yarn_scheduler_minimum_allocation_mb        = 
$::cdh::hadoop::defaults::yarn_scheduler_minimum_allocation_mb,
diff --git a/manifests/hadoop/defaults.pp b/manifests/hadoop/defaults.pp
index 395f1d9..f333455 100644
--- a/manifests/hadoop/defaults.pp
+++ b/manifests/hadoop/defaults.pp
@@ -49,6 +49,7 @@
     # its own.  Adding 0 to it converts it to a Fixnum, which will avoid a
     # 'comparison of String with X faild 'puppet failure.
     $yarn_nodemanager_resource_cpu_vcores        = max($::processorcount - 1, 
1 + 0)
+    $yarn_nodemanager_nofiles_ulimit             = 65536
     $yarn_scheduler_minimum_allocation_mb        = undef
     $yarn_scheduler_maximum_allocation_mb        = undef
     $yarn_scheduler_minimum_allocation_vcores    = undef
diff --git a/manifests/hadoop/nodemanager.pp b/manifests/hadoop/nodemanager.pp
index ee6dcf4..22b659f 100644
--- a/manifests/hadoop/nodemanager.pp
+++ b/manifests/hadoop/nodemanager.pp
@@ -9,6 +9,15 @@
         ensure => 'installed',
     }
 
+    $nofiles_ulimit = $cdh::hadoop::yarn_nodemanager_nofiles_ulimit
+    # Some NodeManager defaults can be overridden
+    file { '/etc/default/hadoop-yarn-nodemanager':
+        content => template('cdh/hadoop/hadoop-yarn-nodemanager.default.erb'),
+        owner   => 'root',
+        group   => 'root',
+        mode    => 0644,
+    }
+
     # Some Hadoop jobs need Zookeeper libraries, but for some reason they
     # are not installed via package dependencies.  Install the CDH
     # zookeeper package here explicitly.  This avoids
diff --git a/templates/hadoop/hadoop-yarn-nodemanager.default.erb 
b/templates/hadoop/hadoop-yarn-nodemanager.default.erb
new file mode 100644
index 0000000..4d8f9b7
--- /dev/null
+++ b/templates/hadoop/hadoop-yarn-nodemanager.default.erb
@@ -0,0 +1,9 @@
+# NOTE: This file is managed by Puppet.
+
+export YARN_IDENT_STRING=yarn
+export YARN_PID_DIR=/var/run/hadoop-yarn
+export YARN_LOG_DIR=/var/log/hadoop-yarn
+export YARN_CONF_DIR=/etc/hadoop/conf
+
+# Set nofiles ulimits for nodemanager processes
+ulimit -n <%= @nofiles_ulimit %>

-- 
To view, visit https://gerrit.wikimedia.org/r/316372
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I0fcfde5acb7358dc7435fb60a494a87f82bfd879
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet/cdh
Gerrit-Branch: master
Gerrit-Owner: Ottomata <o...@wikimedia.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to