Ottomata has submitted this change and it was merged. Change subject: jmxtrans puppetization. ......................................................................
jmxtrans puppetization. Change-Id: I3b8b812b75c188c6e95f42b8342d13350e2e3923 --- A README.md A manifests/init.pp A manifests/metrics.pp A templates/jmxtrans.json.erb 4 files changed, 239 insertions(+), 0 deletions(-) Approvals: Ottomata: Verified; Looks good to me, approved jenkins-bot: Verified diff --git a/README.md b/README.md new file mode 100644 index 0000000..bb62790 --- /dev/null +++ b/README.md @@ -0,0 +1,101 @@ +# puppet-jmxtrans + +A Puppet module for [jmxtrans](https://github.com/jmxtrans/jmxtrans). + +This module assumes that a 'jmxtrans' package is available for +puppet to install. You can build your own from the jmxtrans repository +by running ```ant debian```. + +Use the ```jmxtrans::metrics``` define to install +jmxtrans JSON query config files. See the [jmxtrans wiki](https://github.com/jmxtrans/jmxtrans/wiki/Queries) +if you are not familiar with jmxtrans queries. + +```jmxtrans::metrics``` abstracts much of the repetitive JSON structures +needed to build jmxtrans queries. It currently supports KeyOutWriter, +GangliaWriter and GraphiteWriter. You specify the JMX connection info +and the queries, and configuration information about each output writer +you would like to use. You should use a single ```jmxtrans::metrics``` +define for each JVM you would like query. This will keep JMX queries +to a single JVM bundled together. See jmxtrans +[best practices](https://github.com/jmxtrans/jmxtrans/wiki/BestPractices) +for more information. + +# Usage +## Hadoop NameNode with multiple jmxtrans outputs + +Query a Hadoop NameNode for some stats, and write the metrics to +/tmp/namenode.jmx.out, Ganglia, and Graphite. + +```puppet +jmxtrans::metrics { 'hadoop-hdfs-namenode': + jmx => '127.0.0.1:9980', + outfile => '/tmp/namenode.jmx.out', + ganglia => '127.0.0.1:8649', + ganglia_group_name => 'hadoop', + graphite => '127.0.0.1:2003', + graphite_root_prefix => 'hadoop', + queries => [ + { + 'obj' => 'Hadoop:service=NameNode,name=NameNodeActivity', + 'attr' => ['FileInfoOps', 'FilesCreated', 'FilesDeleted'], + }, + { + 'obj' => 'Hadoop:service=NameNode,name=FSNamesystem', + 'attr' => ['BlockCapacity', 'BlocksTotal', 'TotalFiles'], + }, + ], +} +``` + +## Kafka Broker with jmxtrans Ganglia output + +```puppet +include jmxtrans + +# Since we have multiple hosts sharing the same queries, +# we define a $jmx_kafka_queries variable to hold them. +# This will be passed as the queries parameter to each Kafka host. +$jmx_kafka_queries = [ + { + 'obj' => 'kafka:type=kafka.BrokerAllTopicStat', + 'attr' => [ 'BytesIn', 'BytesOut', 'FailedFetchRequest', 'FailedProduceRequest', 'MessagesIn' ] + }, + { + 'obj' => 'kafka:type=kafka.LogFlushStats', + 'attr' => [ 'AvgFlushMs', 'FlushesPerSecond', 'MaxFlushMs', 'NumFlushes', 'TotalFlushMs' ] + }, + { + 'obj' => 'kafka:type=kafka.SocketServerStats', + 'attr' => [ + 'AvgFetchRequestMs', + 'AvgProduceRequestMs', + 'BytesReadPerSecond', + 'BytesWrittenPerSecond', + 'FetchRequestsPerSecond', + 'MaxFetchRequestMs', + 'MaxProduceRequestMs', + 'NumFetchRequests', + 'NumProduceRequests', + 'ProduceRequestsPerSecond', + 'TotalBytesRead', + 'TotalBytesWritten', + 'TotalFetchRequestMs', + 'TotalProduceRequestMs' + ] + } +] + +# query kafka1 for its JMX metrics +jmxtrans::metrics { 'kafka1': + jmx => 'kafka1:9999', + ganglia => '192.168.10.50:8469', + queries => $jmx_kafka_queries, +} + +# query kafka2 for its JMX metrics +jmxtrans::metrics { 'kafka2': + jmx => 'kafka2:9999', + ganglia => '192.168.10.50:8469', + queries => $jmx_kafka_queries, +} +``` \ No newline at end of file diff --git a/manifests/init.pp b/manifests/init.pp new file mode 100644 index 0000000..b2a3bb2 --- /dev/null +++ b/manifests/init.pp @@ -0,0 +1,16 @@ +# == Class jmxtrans +# Installs a jmxtrans package and ensures that the jmxtrans service is running. +# The jmxtrans::metrics define includes this class, so you probably don't +# need to use it directly. +# +class jmxtrans { + package { 'jmxtrans': + ensure => 'installed', + } + service { 'jmxtrans': + ensure => 'running', + enable => true, + require => Package['jmxtrans'], + } +} + diff --git a/manifests/metrics.pp b/manifests/metrics.pp new file mode 100644 index 0000000..3c2dccf --- /dev/null +++ b/manifests/metrics.pp @@ -0,0 +1,41 @@ +# == Define jmxtrans::metrics +# +# Writes a jmxtrans JSON file in +# /var/lib/jmxtrans/$title.json. +# Each file in this directory should correspond to a single +# JMX instance and specify the object and metric names to query +# the JMX instance for. +# +# == Parameters +# $jmx - host:port of JMX to query (required) +# $queries - array of hashes of the form [ { "obj" => "JMX object name", "attr" => [ "array", "of", "JMX", "metric", "names" ] }, ... ] +# $jmx_alias - Server alias name. Optional. +# $jmx_username - JMX username (if there is one) Optional. +# $jmx_password - JMX password (if there is one) Optional. +# $ganglia - host:port of Ganglia gmond. Optional. +# $ganglia_group_name - Ganglia metrics group. Optional. +# $graphite - host:port of Graphite server Optional. +# $graphite_root_prefix - rootPrefix for Graphite. Optional. +# $outfile - local file path in which to save metric query results. Optional. +# +define jmxtrans::metrics( + $jmx, + $queries, + $jmx_alias = undef, + $jmx_username = undef, + $jmx_password = undef, + $ganglia = undef, + $ganglia_group_name = undef, + $graphite = undef, + $graphite_root_prefix = undef, + $outfile = undef +) +{ + include jmxtrans + + file { "/var/lib/jmxtrans/${title}.json": + content => template('jmxtrans/jmxtrans.json.erb'), + notify => Service['jmxtrans'], + require => Package['jmxtrans'], + } +} \ No newline at end of file diff --git a/templates/jmxtrans.json.erb b/templates/jmxtrans.json.erb new file mode 100644 index 0000000..d268efe --- /dev/null +++ b/templates/jmxtrans.json.erb @@ -0,0 +1,81 @@ +<% + +(jmx_host, jmx_port) = jmx.split(':') + +# Build output writer json strings +# from the outfile, ganglia, and/or +# graphite variables. + +output_writers = [] + +if @outfile + output_writers.push("{ + \"@class\" : \"com.googlecode.jmxtrans.model.output.KeyOutWriter\", + \"settings\" : { + \"outputFile\" : \"#{outfile}\" + } + }") +end + +if @ganglia + (ganglia_host, ganglia_port) = ganglia.split(':') if @ganglia + + ganglia_output_writer = "{ + \"@class\" : \"com.googlecode.jmxtrans.model.output.GangliaWriter\", + \"settings\" : { + \"host\" : \"#{ganglia_host}\", + \"port\" : #{ganglia_port}" + ganglia_output_writer += ",\n \"groupName\" : \"#{ganglia_group_name}\"" if @ganglia_group_name + ganglia_output_writer += "\n }\n }" + output_writers.push(ganglia_output_writer) +end + +if @graphite + (graphite_host, graphite_port) = graphite.split(':') + graphite_output_writer = "{ + \"@class\" : \"com.googlecode.jmxtrans.model.output.GraphiteWriter\", + \"settings\" : { + \"host\" : \"#{graphite_host}\", + \"port\" : #{graphite_port}" + graphite_output_writer += ",\n \"rootPrefix\" : \"#{graphite_root_prefix}\"" if @graphite_root_prefix + graphite_output_writer += "\n }\n }" + output_writers.push(graphite_output_writer) +end + +# Output the JSON string. +-%> + +{ "servers" : [ { + + "host" : "<%= jmx_host %>", + "port" : <%= jmx_port %>, +<% if @jmx_alias -%> + "alias" : "<%= @jmx_alias %>", +<% end -%> +<% if @jmx_username -%> + "username" : "<%= @jmx_username %>", +<% end -%> +<% if @jmx_password -%> + "password" : "<%= @jmx_password %>", +<% end -%> + "queries" : + [ +<% # loop over each query and output it with each configured output_writer -%> +<% queries.each_with_index do |query, index| -%> +<% unless index == 0 -%> + , +<% end -%> + { + "obj" : "<%= query["obj"] %>", + "attr" : [ <%= query['attr'].map {|t| "\""+t.to_s+"\""}.join(', ') %> ], +<% if query.has_key? "resultAlias" -%> + "resultAlias" : "<%= query["resultAlias"] %>", +<% end -%> + "outputWriters" : [ + <%= output_writers.join(",\n ") %> + ] + } +<% end # queries.each_with_index -%> + ] + +} ] } -- To view, visit https://gerrit.wikimedia.org/r/70915 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I3b8b812b75c188c6e95f42b8342d13350e2e3923 Gerrit-PatchSet: 3 Gerrit-Project: operations/puppet/jmxtrans Gerrit-Branch: master Gerrit-Owner: Ottomata <[email protected]> Gerrit-Reviewer: Akosiaris <[email protected]> Gerrit-Reviewer: Faidon <[email protected]> Gerrit-Reviewer: Hashar <[email protected]> Gerrit-Reviewer: Manybubbles <[email protected]> Gerrit-Reviewer: Ottomata <[email protected]> Gerrit-Reviewer: jenkins-bot _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
