Ottomata has uploaded a new change for review. https://gerrit.wikimedia.org/r/67419
Change subject: Adding support for ganglia metrics in hadoop-metrics2.properties ...................................................................... Adding support for ganglia metrics in hadoop-metrics2.properties Change-Id: I547a9c0faaeb2d937c506f57a271230c4fe64c06 --- M TODO.md M manifests/hadoop.pp M manifests/hadoop/defaults.pp A templates/hadoop/hadoop-metrics2.properties.erb 4 files changed, 62 insertions(+), 2 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet/cdh4 refs/changes/19/67419/1 diff --git a/TODO.md b/TODO.md index 9dac694..04e2dd2 100644 --- a/TODO.md +++ b/TODO.md @@ -2,7 +2,6 @@ ## Hadoop -- Add hadoop-metrics2.properties configuration - Add hosts.exclude support for decommissioning nodes. - Change cluster (conf) name? (use update-alternatives?) - Set default map/reduce tasks automatically based on facter node stats. @@ -12,6 +11,7 @@ - Support Secondary NameNode. - Support High Availability NameNode. - Make JMX ports configurable. +- Make hadoop-metrics2.properties more configurable. ## Hive - Hive Server + Hive Metastore @@ -24,3 +24,6 @@ ## Zookeeper +Won't implement. A Zookeeper package is available upstream in Debian/Ubuntu. +Puppetization for this package can be found at +https://github.com/wikimedia/operations-puppet-zookeeper diff --git a/manifests/hadoop.pp b/manifests/hadoop.pp index 49562a0..e974daf 100644 --- a/manifests/hadoop.pp +++ b/manifests/hadoop.pp @@ -37,6 +37,7 @@ # $yarn_nodemanager_resource_memory_mb # $yarn_resourcemanager_scheduler_class - If you change this (e.g. to FairScheduler), you should also provide your own scheduler config .xml files outside of the cdh4 module. # $use_yarn +# $ganglia_hosts - Set this to an array of ganglia host:ports if you want to enable ganglia sinks in hadoop-metrics2.properites # class cdh4::hadoop( $namenode_hostname, @@ -64,7 +65,8 @@ $mapreduce_final_compession = $::cdh4::hadoop::defaults::mapreduce_final_compession, $yarn_nodemanager_resource_memory_mb = $::cdh4::hadoop::defaults::yarn_nodemanager_resource_memory_mb, $yarn_resourcemanager_scheduler_class = $::cdh4::hadoop::defaults::yarn_resourcemanager_scheduler_class, - $use_yarn = $::cdh4::hadoop::defaults::use_yarn + $use_yarn = $::cdh4::hadoop::defaults::use_yarn, + $ganglia_hosts = $::cdh4::hadoop::defaults::ganglia_hosts, ) inherits cdh4::hadoop::defaults { @@ -129,4 +131,15 @@ ensure => $yarn_ensure, content => template('cdh4/hadoop/yarn-env.sh.erb'), } + + # render hadoop-metrics2.properties + # if we hav Ganglia Hosts to send metrics to. + $hadoop_metrics2_ensure = $ganglia_hosts ? { + undef => 'absent', + default => 'present', + } + file { "${config_directory}/hadoop-metrics2.properties": + ensure => $hadoop_metrics2_ensure, + content => template('cdh4/hadoop/hadoop-metrics2.properties.erb'), + } } diff --git a/manifests/hadoop/defaults.pp b/manifests/hadoop/defaults.pp index 3fc6c5d..373ca8e 100644 --- a/manifests/hadoop/defaults.pp +++ b/manifests/hadoop/defaults.pp @@ -26,4 +26,5 @@ $yarn_nodemanager_resource_memory_mb = undef $yarn_resourcemanager_scheduler_class = undef $use_yarn = true + $ganglia_hosts = undef } \ No newline at end of file diff --git a/templates/hadoop/hadoop-metrics2.properties.erb b/templates/hadoop/hadoop-metrics2.properties.erb new file mode 100644 index 0000000..9075e62 --- /dev/null +++ b/templates/hadoop/hadoop-metrics2.properties.erb @@ -0,0 +1,43 @@ +# NOTE: This file is managed by Puppet. + +# syntax: [prefix].[source|sink].[instance].[options] +# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details + +# default sampling period, in seconds +*.period=10 + +<% if @ganglia_hosts +ganglia_hosts_string = ganglia_hosts.sort.join(',') +-%> +# +# Below are for sending metrics to Ganglia +# + +# for Ganglia 3.1 support +*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31 + +*.sink.ganglia.period=10 + +# default for supportsparse is false +# *.sink.ganglia.supportsparse=true + +*.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both +*.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 + +namenode.sink.ganglia.servers=<%= ganglia_hosts_string %> +datanode.sink.ganglia.servers=<%= ganglia_hosts_string %> + +<% if use_yarn -%> +resourcemanager.sink.ganglia.servers=<%= ganglia_hosts_string %> +nodemanager.sink.ganglia.servers=<%= ganglia_hosts_string %> +<% else -%> +jobtracker.sink.ganglia.servers=<%= ganglia_hosts_string %> +tasktracker.sink.ganglia.servers=<%= ganglia_hosts_string %> +<% end -%> + +maptask.sink.ganglia.servers=<%= ganglia_hosts_string %> +reducetask.sink.ganglia.servers=<%= ganglia_hosts_string %> + +secondarynamenode.sink.ganglia.servers=<%= ganglia_hosts_string %> + +<% end -%> \ No newline at end of file -- To view, visit https://gerrit.wikimedia.org/r/67419 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I547a9c0faaeb2d937c506f57a271230c4fe64c06 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet/cdh4 Gerrit-Branch: master Gerrit-Owner: Ottomata <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
