[MediaWiki-commits] [Gerrit] operations...cdh[master]: Override nofiles ulimit for nodemanager processes

2016-10-17 Thread Ottomata (Code Review) 
Ottomata has submitted this change and it was merged.

Change subject: Override nofiles ulimit for nodemanager processes
..


Override nofiles ulimit for nodemanager processes

Bug: T148206
Change-Id: I0fcfde5acb7358dc7435fb60a494a87f82bfd879
---
M manifests/hadoop.pp
M manifests/hadoop/defaults.pp
M manifests/hadoop/nodemanager.pp
A templates/hadoop/hadoop-yarn-nodemanager.default.erb
4 files changed, 21 insertions(+), 0 deletions(-)

Approvals:
  Ottomata: Looks good to me, approved
  jenkins-bot: Verified



diff --git a/manifests/hadoop.pp b/manifests/hadoop.pp
index 079d687..86bfc7e 100644
--- a/manifests/hadoop.pp
+++ b/manifests/hadoop.pp
@@ -77,6 +77,7 @@
 #   Default: RECORD
 #   $yarn_nodemanager_resource_memory_mb
 #   $yarn_nodemanager_resource_cpu_vcores - Default: max($::processorcount 
- 1, 1)
+#   $yarn_nodemanager_nofiles_ulimit  - Default: 65536
 #   $yarn_scheduler_minimum_allocation_mb - The minimum allocation for 
every container request at the RM,
 #   in MBs. Memory requests lower 
than this won't take effect, and
 #   the specified value will get 
allocated at minimum.
@@ -167,6 +168,7 @@
 $mapreduce_output_compression_type   = 
$::cdh::hadoop::defaults::mapreduce_output_compression_type,
 $yarn_nodemanager_resource_memory_mb = 
$::cdh::hadoop::defaults::yarn_nodemanager_resource_memory_mb,
 $yarn_nodemanager_resource_cpu_vcores= 
$::cdh::hadoop::defaults::yarn_nodemanager_resource_cpu_vcores,
+$yarn_nodemanager_nofiles_ulimit = 
$::cdh::hadoop::defaults::yarn_nodemanager_nofiles_ulimit,
 $yarn_log_aggregation_retain_seconds = 
$::cdh::hadoop::defaults::yarn_log_aggregation_retain_seconds,
 $yarn_log_aggregation_retain_check_interval_seconds = 
$::cdh::hadoop::defaults::yarn_log_aggregation_retain_check_interval_seconds,
 $yarn_scheduler_minimum_allocation_mb= 
$::cdh::hadoop::defaults::yarn_scheduler_minimum_allocation_mb,
diff --git a/manifests/hadoop/defaults.pp b/manifests/hadoop/defaults.pp
index 395f1d9..f333455 100644
--- a/manifests/hadoop/defaults.pp
+++ b/manifests/hadoop/defaults.pp
@@ -49,6 +49,7 @@
 # its own.  Adding 0 to it converts it to a Fixnum, which will avoid a
 # 'comparison of String with X faild 'puppet failure.
 $yarn_nodemanager_resource_cpu_vcores= max($::processorcount - 1, 
1 + 0)
+$yarn_nodemanager_nofiles_ulimit = 65536
 $yarn_scheduler_minimum_allocation_mb= undef
 $yarn_scheduler_maximum_allocation_mb= undef
 $yarn_scheduler_minimum_allocation_vcores= undef
diff --git a/manifests/hadoop/nodemanager.pp b/manifests/hadoop/nodemanager.pp
index ee6dcf4..22b659f 100644
--- a/manifests/hadoop/nodemanager.pp
+++ b/manifests/hadoop/nodemanager.pp
@@ -9,6 +9,15 @@
 ensure => 'installed',
 }
 
+$nofiles_ulimit = $cdh::hadoop::yarn_nodemanager_nofiles_ulimit
+# Some NodeManager defaults can be overridden
+file { '/etc/default/hadoop-yarn-nodemanager':
+content => template('cdh/hadoop/hadoop-yarn-nodemanager.default.erb'),
+owner   => 'root',
+group   => 'root',
+mode=> 0644,
+}
+
 # Some Hadoop jobs need Zookeeper libraries, but for some reason they
 # are not installed via package dependencies.  Install the CDH
 # zookeeper package here explicitly.  This avoids
diff --git a/templates/hadoop/hadoop-yarn-nodemanager.default.erb 
b/templates/hadoop/hadoop-yarn-nodemanager.default.erb
new file mode 100644
index 000..4d8f9b7
--- /dev/null
+++ b/templates/hadoop/hadoop-yarn-nodemanager.default.erb
@@ -0,0 +1,9 @@
+# NOTE: This file is managed by Puppet.
+
+export YARN_IDENT_STRING=yarn
+export YARN_PID_DIR=/var/run/hadoop-yarn
+export YARN_LOG_DIR=/var/log/hadoop-yarn
+export YARN_CONF_DIR=/etc/hadoop/conf
+
+# Set nofiles ulimits for nodemanager processes
+ulimit -n <%= @nofiles_ulimit %>

-- 
To view, visit https://gerrit.wikimedia.org/r/316372
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I0fcfde5acb7358dc7435fb60a494a87f82bfd879
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet/cdh
Gerrit-Branch: master
Gerrit-Owner: Ottomata 
Gerrit-Reviewer: Elukey 
Gerrit-Reviewer: Ottomata 
Gerrit-Reviewer: jenkins-bot <>

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations...cdh[master]: Override nofiles ulimit for nodemanager processes

2016-10-17 Thread Ottomata (Code Review) 
Ottomata has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/316372

Change subject: Override nofiles ulimit for nodemanager processes
..

Override nofiles ulimit for nodemanager processes

Bug: T148206
Change-Id: I0fcfde5acb7358dc7435fb60a494a87f82bfd879
---
M manifests/hadoop.pp
M manifests/hadoop/defaults.pp
M manifests/hadoop/nodemanager.pp
A templates/hadoop/hadoop-yarn-nodemanager.default.erb
4 files changed, 21 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet/cdh 
refs/changes/72/316372/1

diff --git a/manifests/hadoop.pp b/manifests/hadoop.pp
index 079d687..86bfc7e 100644
--- a/manifests/hadoop.pp
+++ b/manifests/hadoop.pp
@@ -77,6 +77,7 @@
 #   Default: RECORD
 #   $yarn_nodemanager_resource_memory_mb
 #   $yarn_nodemanager_resource_cpu_vcores - Default: max($::processorcount 
- 1, 1)
+#   $yarn_nodemanager_nofiles_ulimit  - Default: 65536
 #   $yarn_scheduler_minimum_allocation_mb - The minimum allocation for 
every container request at the RM,
 #   in MBs. Memory requests lower 
than this won't take effect, and
 #   the specified value will get 
allocated at minimum.
@@ -167,6 +168,7 @@
 $mapreduce_output_compression_type   = 
$::cdh::hadoop::defaults::mapreduce_output_compression_type,
 $yarn_nodemanager_resource_memory_mb = 
$::cdh::hadoop::defaults::yarn_nodemanager_resource_memory_mb,
 $yarn_nodemanager_resource_cpu_vcores= 
$::cdh::hadoop::defaults::yarn_nodemanager_resource_cpu_vcores,
+$yarn_nodemanager_nofiles_ulimit = 
$::cdh::hadoop::defaults::yarn_nodemanager_nofiles_ulimit,
 $yarn_log_aggregation_retain_seconds = 
$::cdh::hadoop::defaults::yarn_log_aggregation_retain_seconds,
 $yarn_log_aggregation_retain_check_interval_seconds = 
$::cdh::hadoop::defaults::yarn_log_aggregation_retain_check_interval_seconds,
 $yarn_scheduler_minimum_allocation_mb= 
$::cdh::hadoop::defaults::yarn_scheduler_minimum_allocation_mb,
diff --git a/manifests/hadoop/defaults.pp b/manifests/hadoop/defaults.pp
index 395f1d9..f333455 100644
--- a/manifests/hadoop/defaults.pp
+++ b/manifests/hadoop/defaults.pp
@@ -49,6 +49,7 @@
 # its own.  Adding 0 to it converts it to a Fixnum, which will avoid a
 # 'comparison of String with X faild 'puppet failure.
 $yarn_nodemanager_resource_cpu_vcores= max($::processorcount - 1, 
1 + 0)
+$yarn_nodemanager_nofiles_ulimit = 65536
 $yarn_scheduler_minimum_allocation_mb= undef
 $yarn_scheduler_maximum_allocation_mb= undef
 $yarn_scheduler_minimum_allocation_vcores= undef
diff --git a/manifests/hadoop/nodemanager.pp b/manifests/hadoop/nodemanager.pp
index ee6dcf4..22b659f 100644
--- a/manifests/hadoop/nodemanager.pp
+++ b/manifests/hadoop/nodemanager.pp
@@ -9,6 +9,15 @@
 ensure => 'installed',
 }
 
+$nofiles_ulimit = $cdh::hadoop::yarn_nodemanager_nofiles_ulimit
+# Some NodeManager defaults can be overridden
+file { '/etc/default/hadoop-yarn-nodemanager':
+content => template('cdh/hadoop/hadoop-yarn-nodemanager.default.erb'),
+owner   => 'root',
+group   => 'root',
+mode=> 0644,
+}
+
 # Some Hadoop jobs need Zookeeper libraries, but for some reason they
 # are not installed via package dependencies.  Install the CDH
 # zookeeper package here explicitly.  This avoids
diff --git a/templates/hadoop/hadoop-yarn-nodemanager.default.erb 
b/templates/hadoop/hadoop-yarn-nodemanager.default.erb
new file mode 100644
index 000..4d8f9b7
--- /dev/null
+++ b/templates/hadoop/hadoop-yarn-nodemanager.default.erb
@@ -0,0 +1,9 @@
+# NOTE: This file is managed by Puppet.
+
+export YARN_IDENT_STRING=yarn
+export YARN_PID_DIR=/var/run/hadoop-yarn
+export YARN_LOG_DIR=/var/log/hadoop-yarn
+export YARN_CONF_DIR=/etc/hadoop/conf
+
+# Set nofiles ulimits for nodemanager processes
+ulimit -n <%= @nofiles_ulimit %>

-- 
To view, visit https://gerrit.wikimedia.org/r/316372
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I0fcfde5acb7358dc7435fb60a494a87f82bfd879
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet/cdh
Gerrit-Branch: master
Gerrit-Owner: Ottomata 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits