Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/270027
Change subject: Fix hadoop_namenode_opts param, move hadoop logstash logging
configs into separate class
......................................................................
Fix hadoop_namenode_opts param, move hadoop logstash logging configs into
separate class
Change-Id: I11aba4281a5fcfdbd532d84defe1985026549fb9
---
M hieradata/eqiad/cdh/hadoop.yaml
M modules/role/manifests/analytics_cluster/hadoop/client.pp
A modules/role/manifests/analytics_cluster/hadoop/logstash.pp
3 files changed, 55 insertions(+), 39 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/27/270027/1
diff --git a/hieradata/eqiad/cdh/hadoop.yaml b/hieradata/eqiad/cdh/hadoop.yaml
index 932c520..be8fa55 100644
--- a/hieradata/eqiad/cdh/hadoop.yaml
+++ b/hieradata/eqiad/cdh/hadoop.yaml
@@ -32,7 +32,7 @@
net_topology_script_template:
'role/analytics_cluster/hadoop/net-topology.py.erb'
# Increase NameNode heapsize independent from other daemons
-namenode_opts: "-Xmx4096m"
+hadoop_namenode_opts: "-Xmx4096m"
# Ensure that users in these groups have home directories in HDFS.
hadoop_users_posix_groups: "analytics-users analytics-privatedata-users
analytics-admins analytics-search-users"
diff --git a/modules/role/manifests/analytics_cluster/hadoop/client.pp
b/modules/role/manifests/analytics_cluster/hadoop/client.pp
index 983484e..d1caff2 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/client.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/client.pp
@@ -64,6 +64,9 @@
'yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage'
=> '99.0',
},
}
+ # This will only enable logstash logging if
+ # $cdh::hadoop::gelf_logging_enabled is true.
+ include role::analytics_cluster::hadoop::logstash
# If in production AND the current node is a journalnode, then
# go ahead and include an icinga alert for the JournalNode process.
@@ -73,44 +76,6 @@
nrpe_command => '/usr/lib/nagios/plugins/check_procs -c 1:1 -C
java -a "org.apache.hadoop.hdfs.qjournal.server.JournalNode"',
require => Class['cdh::hadoop'],
critical => true,
- }
- }
-
- file { '/usr/local/bin/hadoop-yarn-logging-helper.sh':
- content => template('hadoop/hadoop-yarn-logging-helper.erb'),
- mode => '0744',
- }
- if $::cdh::hadoop::gelf_logging_enabled {
- ensure_packages([
- # library dependency
- 'libjson-simple-java',
- # the libary itself: logstash-gelf.jar
- 'liblogstash-gelf-java',
- ])
- # symlink into hadoop classpath
- file { '/usr/lib/hadoop/lib/json_simple.jar':
- ensure => 'link',
- target => '/usr/share/java/json_simple.jar',
- require => Package['libjson-simple-java'],
- }
-
- # symlink into hadoop classpath
- file { '/usr/lib/hadoop/lib/logstash-gelf.jar':
- ensure => 'link',
- target => '/usr/share/java/logstash-gelf.jar',
- require => Package['liblogstash-gelf-java'],
- }
- # Patch container-log4j.properties inside nodemanager jar
- # See script source for details
- exec { 'hadoop-yarn-logging-helper-set':
- command => '/usr/local/bin/hadoop-yarn-logging-helper.sh set',
- subscribe => File['/usr/local/bin/hadoop-yarn-logging-helper.sh'],
- }
- } else {
- # Revert to original unmodified jar
- exec { 'hadoop-yarn-logging-helper-reset':
- command => '/usr/local/bin/hadoop-yarn-logging-helper.sh reset',
- subscribe => File['/usr/local/bin/hadoop-yarn-logging-helper.sh'],
}
}
diff --git a/modules/role/manifests/analytics_cluster/hadoop/logstash.pp
b/modules/role/manifests/analytics_cluster/hadoop/logstash.pp
new file mode 100644
index 0000000..6591745
--- /dev/null
+++ b/modules/role/manifests/analytics_cluster/hadoop/logstash.pp
@@ -0,0 +1,51 @@
+# == Class role::analytics_cluster::hadoop::logstash
+# Enables gelf logging to logstash from Hadoop.
+# As of 2016-02, this is not used.
+class role::analytics_cluster::hadoop::logstash {
+ Class['cdh::hadoop'] -> Class['role::analytics_cluster::hadoop::logstash']
+
+ file { '/usr/local/bin/hadoop-yarn-logging-helper.sh':
+ content => template('hadoop/hadoop-yarn-logging-helper.erb'),
+ mode => '0744',
+ }
+
+ $patched_jar_exists_command = '/bin/ls /usr/lib/hadoop-yarn |
/usr/bin/grep -E "hadoop-yarn-server-nodemanager.+gelf"'
+
+ if $cdh::hadoop::gelf_logging_enabled {
+ ensure_packages([
+ # library dependency
+ 'libjson-simple-java',
+ # the libary itself: logstash-gelf.jar
+ 'liblogstash-gelf-java',
+ ])
+ # symlink into hadoop classpath
+ file { '/usr/lib/hadoop/lib/json_simple.jar':
+ ensure => 'link',
+ target => '/usr/share/java/json_simple.jar',
+ require => Package['libjson-simple-java'],
+ }
+
+ # symlink into hadoop classpath
+ file { '/usr/lib/hadoop/lib/logstash-gelf.jar':
+ ensure => 'link',
+ target => '/usr/share/java/logstash-gelf.jar',
+ require => Package['liblogstash-gelf-java'],
+ }
+
+ # Patch container-log4j.properties inside nodemanager jar
+ # See script source for details
+ exec { 'hadoop-yarn-logging-helper-set':
+ command => '/usr/local/bin/hadoop-yarn-logging-helper.sh set',
+ subscribe => File['/usr/local/bin/hadoop-yarn-logging-helper.sh'],
+ unless => $patched_jar_exists_command,
+ }
+ }
+ else {
+ # Revert to original unmodified jar
+ exec { 'hadoop-yarn-logging-helper-reset':
+ command => '/usr/local/bin/hadoop-yarn-logging-helper.sh reset',
+ subscribe => File['/usr/local/bin/hadoop-yarn-logging-helper.sh'],
+ onlyif => $patched_jar_exists_command,
+ }
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/270027
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I11aba4281a5fcfdbd532d84defe1985026549fb9
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
