Ottomata has submitted this change and it was merged. Change subject: Override nofiles ulimit for nodemanager processes ......................................................................
Override nofiles ulimit for nodemanager processes Bug: T148206 Change-Id: I0fcfde5acb7358dc7435fb60a494a87f82bfd879 --- M manifests/hadoop.pp M manifests/hadoop/defaults.pp M manifests/hadoop/nodemanager.pp A templates/hadoop/hadoop-yarn-nodemanager.default.erb 4 files changed, 21 insertions(+), 0 deletions(-) Approvals: Ottomata: Looks good to me, approved jenkins-bot: Verified diff --git a/manifests/hadoop.pp b/manifests/hadoop.pp index 079d687..86bfc7e 100644 --- a/manifests/hadoop.pp +++ b/manifests/hadoop.pp @@ -77,6 +77,7 @@ # Default: RECORD # $yarn_nodemanager_resource_memory_mb # $yarn_nodemanager_resource_cpu_vcores - Default: max($::processorcount - 1, 1) +# $yarn_nodemanager_nofiles_ulimit - Default: 65536 # $yarn_scheduler_minimum_allocation_mb - The minimum allocation for every container request at the RM, # in MBs. Memory requests lower than this won't take effect, and # the specified value will get allocated at minimum. @@ -167,6 +168,7 @@ $mapreduce_output_compression_type = $::cdh::hadoop::defaults::mapreduce_output_compression_type, $yarn_nodemanager_resource_memory_mb = $::cdh::hadoop::defaults::yarn_nodemanager_resource_memory_mb, $yarn_nodemanager_resource_cpu_vcores = $::cdh::hadoop::defaults::yarn_nodemanager_resource_cpu_vcores, + $yarn_nodemanager_nofiles_ulimit = $::cdh::hadoop::defaults::yarn_nodemanager_nofiles_ulimit, $yarn_log_aggregation_retain_seconds = $::cdh::hadoop::defaults::yarn_log_aggregation_retain_seconds, $yarn_log_aggregation_retain_check_interval_seconds = $::cdh::hadoop::defaults::yarn_log_aggregation_retain_check_interval_seconds, $yarn_scheduler_minimum_allocation_mb = $::cdh::hadoop::defaults::yarn_scheduler_minimum_allocation_mb, diff --git a/manifests/hadoop/defaults.pp b/manifests/hadoop/defaults.pp index 395f1d9..f333455 100644 --- a/manifests/hadoop/defaults.pp +++ b/manifests/hadoop/defaults.pp @@ -49,6 +49,7 @@ # its own. Adding 0 to it converts it to a Fixnum, which will avoid a # 'comparison of String with X faild 'puppet failure. $yarn_nodemanager_resource_cpu_vcores = max($::processorcount - 1, 1 + 0) + $yarn_nodemanager_nofiles_ulimit = 65536 $yarn_scheduler_minimum_allocation_mb = undef $yarn_scheduler_maximum_allocation_mb = undef $yarn_scheduler_minimum_allocation_vcores = undef diff --git a/manifests/hadoop/nodemanager.pp b/manifests/hadoop/nodemanager.pp index ee6dcf4..22b659f 100644 --- a/manifests/hadoop/nodemanager.pp +++ b/manifests/hadoop/nodemanager.pp @@ -9,6 +9,15 @@ ensure => 'installed', } + $nofiles_ulimit = $cdh::hadoop::yarn_nodemanager_nofiles_ulimit + # Some NodeManager defaults can be overridden + file { '/etc/default/hadoop-yarn-nodemanager': + content => template('cdh/hadoop/hadoop-yarn-nodemanager.default.erb'), + owner => 'root', + group => 'root', + mode => 0644, + } + # Some Hadoop jobs need Zookeeper libraries, but for some reason they # are not installed via package dependencies. Install the CDH # zookeeper package here explicitly. This avoids diff --git a/templates/hadoop/hadoop-yarn-nodemanager.default.erb b/templates/hadoop/hadoop-yarn-nodemanager.default.erb new file mode 100644 index 0000000..4d8f9b7 --- /dev/null +++ b/templates/hadoop/hadoop-yarn-nodemanager.default.erb @@ -0,0 +1,9 @@ +# NOTE: This file is managed by Puppet. + +export YARN_IDENT_STRING=yarn +export YARN_PID_DIR=/var/run/hadoop-yarn +export YARN_LOG_DIR=/var/log/hadoop-yarn +export YARN_CONF_DIR=/etc/hadoop/conf + +# Set nofiles ulimits for nodemanager processes +ulimit -n <%= @nofiles_ulimit %> -- To view, visit https://gerrit.wikimedia.org/r/316372 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I0fcfde5acb7358dc7435fb60a494a87f82bfd879 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet/cdh Gerrit-Branch: master Gerrit-Owner: Ottomata <o...@wikimedia.org> Gerrit-Reviewer: Elukey <ltosc...@wikimedia.org> Gerrit-Reviewer: Ottomata <o...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits