Elukey has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/395054 )
Change subject: role::hadoop::master|standby: add Prometheus JMX exporter config
......................................................................
role::hadoop::master|standby: add Prometheus JMX exporter config
Bug: T177458
Change-Id: Ie79831b65b8f03f6f77e495614c2276614f7dd97
---
M hieradata/role/common/analytics_cluster/hadoop/master.yaml
M hieradata/role/common/analytics_cluster/hadoop/standby.yaml
A modules/profile/files/hadoop/prometheus_hdfs_namenode_jmx_exporter.yaml
A modules/profile/files/hadoop/prometheus_mapreduce_history_jmx_exporter.yaml
A modules/profile/files/hadoop/prometheus_yarn_resourcemanager_jmx_exporter.yaml
M modules/profile/manifests/hadoop/master.pp
M modules/profile/manifests/hadoop/master/standby.pp
A modules/profile/manifests/hadoop/monitoring/history.pp
A modules/profile/manifests/hadoop/monitoring/namenode.pp
A modules/profile/manifests/hadoop/monitoring/resourcemanager.pp
10 files changed, 94 insertions(+), 8 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/54/395054/1
diff --git a/hieradata/role/common/analytics_cluster/hadoop/master.yaml
b/hieradata/role/common/analytics_cluster/hadoop/master.yaml
index c595c4d..bf5ee28 100644
--- a/hieradata/role/common/analytics_cluster/hadoop/master.yaml
+++ b/hieradata/role/common/analytics_cluster/hadoop/master.yaml
@@ -17,11 +17,12 @@
# This setting is used to configure the max heap size for both.
# The default is 1000m, we increase it to 2048m in production.
profile::hadoop::common::yarn_heapsize: 2048
-profile::hadoop::common::yarn_resourcemanager_opts: '-Xms2048m'
-profile::hadoop::common::hadoop_namenode_opts: "-Xms6144m -Xmx6144m"
# Used to set up jvm heap size usage thresholds
profile::hadoop::master::namenode_heapsize: 6144
-# MapReduce History server (running on Hadoop master nodes)
-profile::hadoop::common::mapreduce_history_java_opts: '-Xms4096m -Xmx4096m'
\ No newline at end of file
+# Prometheus JMX Exporter config templates. The missing variables will be
filled
+# in the worker profile before commons.
+profile::hadoop::common::hadoop_namenode_opts: "-Xms6144m -Xmx6144m
-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=%{::ipaddress}:%{::profile::hadoop::master::prometheus_jmx_exporter_namenode_port}:%{::profile::hadoop::master::namenode_jmx_exporter_config_file}"
+profile::hadoop::common::yarn_resourcemanager_opts: "-Xms2048m
-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=%{::ipaddress}:%{::profile::hadoop::master::prometheus_jmx_exporter_resourcemanager_port}:%{::profile::hadoop::master::resourcemanager_jmx_exporter_config_file}"
+profile::hadoop::common::mapreduce_history_java_opts: "-Xms4096m -Xmx4096m
-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=%{::ipaddress}:%{::profile::hadoop::master::prometheus_jmx_exporter_history_port}:%{::profile::hadoop::master::history_jmx_exporter_config_file}"
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/hadoop/standby.yaml
b/hieradata/role/common/analytics_cluster/hadoop/standby.yaml
index 654a0e2..26e217e 100644
--- a/hieradata/role/common/analytics_cluster/hadoop/standby.yaml
+++ b/hieradata/role/common/analytics_cluster/hadoop/standby.yaml
@@ -21,11 +21,11 @@
# This setting is used to configure the max heap size for both.
# The default is 1000m, we increase it to 2048m in production.
profile::hadoop::common::yarn_heapsize: 2048
-profile::hadoop::common::yarn_resourcemanager_opts: '-Xms2048m'
-profile::hadoop::common::hadoop_namenode_opts: "-Xms6144m -Xmx6144m"
# Used to set up jvm heap size usage thresholds
profile::hadoop::standby::namenode_heapsize: 6144
-# MapReduce History server (running on Hadoop master nodes)
-profile::hadoop::common::mapreduce_history_java_opts: '-Xms4096m -Xmx4096m'
\ No newline at end of file
+# Prometheus JMX Exporter config templates. The missing variables will be
filled
+# in the worker profile before commons.
+profile::hadoop::common::hadoop_namenode_opts: "-Xms6144m -Xmx6144m
-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=%{::ipaddress}:%{::profile::hadoop::master::prometheus_jmx_exporter_namenode_port}:%{::profile::hadoop::master::namenode_jmx_exporter_config_file}"
+profile::hadoop::common::yarn_resourcemanager_opts: "-Xms2048m
-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=%{::ipaddress}:%{::profile::hadoop::master::prometheus_jmx_exporter_resourcemanager_port}:%{::profile::hadoop::master::resourcemanager_jmx_exporter_config_file}
\ No newline at end of file
diff --git
a/modules/profile/files/hadoop/prometheus_hdfs_namenode_jmx_exporter.yaml
b/modules/profile/files/hadoop/prometheus_hdfs_namenode_jmx_exporter.yaml
new file mode 100644
index 0000000..73530b2
--- /dev/null
+++ b/modules/profile/files/hadoop/prometheus_hdfs_namenode_jmx_exporter.yaml
@@ -0,0 +1,3 @@
+---
+lowercaseOutputLabelNames: true
+lowercaseOutputName: false
\ No newline at end of file
diff --git
a/modules/profile/files/hadoop/prometheus_mapreduce_history_jmx_exporter.yaml
b/modules/profile/files/hadoop/prometheus_mapreduce_history_jmx_exporter.yaml
new file mode 100644
index 0000000..73530b2
--- /dev/null
+++
b/modules/profile/files/hadoop/prometheus_mapreduce_history_jmx_exporter.yaml
@@ -0,0 +1,3 @@
+---
+lowercaseOutputLabelNames: true
+lowercaseOutputName: false
\ No newline at end of file
diff --git
a/modules/profile/files/hadoop/prometheus_yarn_resourcemanager_jmx_exporter.yaml
b/modules/profile/files/hadoop/prometheus_yarn_resourcemanager_jmx_exporter.yaml
new file mode 100644
index 0000000..73530b2
--- /dev/null
+++
b/modules/profile/files/hadoop/prometheus_yarn_resourcemanager_jmx_exporter.yaml
@@ -0,0 +1,3 @@
+---
+lowercaseOutputLabelNames: true
+lowercaseOutputName: false
\ No newline at end of file
diff --git a/modules/profile/manifests/hadoop/master.pp
b/modules/profile/manifests/hadoop/master.pp
index e5d06ea..fb0c384 100644
--- a/modules/profile/manifests/hadoop/master.pp
+++ b/modules/profile/manifests/hadoop/master.pp
@@ -65,6 +65,19 @@
# Include icinga alerts if production realm.
if $monitoring_enabled {
+ # Prometheus exporters
+ include ::profile::hadoop::monitoring::namenode
+ $namenode_jmx_exporter_config_file =
$::profile::hadoop::monitoring::namenode::jmx_exporter_config_file
+ $prometheus_jmx_exporter_namenode_port =
$::profile::hadoop::monitoring::datanode::prometheus_jmx_exporter_namenode_port
+
+ include ::profile::hadoop::monitoring::resourcemanager
+ $resourcemanager_jmx_exporter_config_file =
$::profile::hadoop::monitoring::resourcemanager::jmx_exporter_config_file
+ $prometheus_jmx_exporter_resourcemanager_port =
$::profile::hadoop::monitoring::resourcemanager::prometheus_jmx_exporter_resourcemanager_port
+
+ include ::profile::hadoop::monitoring::history
+ $history_jmx_exporter_config_file =
$::profile::hadoop::monitoring::history::jmx_exporter_config_file
+ $prometheus_jmx_exporter_history_port =
$::profile::hadoop::monitoring::history::prometheus_jmx_exporter_history_port
+
# Icinga process alerts for NameNode, ResourceManager and HistoryServer
nrpe::monitor_service { 'hadoop-hdfs-namenode':
description => 'Hadoop Namenode - Primary',
diff --git a/modules/profile/manifests/hadoop/master/standby.pp
b/modules/profile/manifests/hadoop/master/standby.pp
index 9d02235..2ecb30a 100644
--- a/modules/profile/manifests/hadoop/master/standby.pp
+++ b/modules/profile/manifests/hadoop/master/standby.pp
@@ -24,6 +24,15 @@
# Include icinga alerts if production realm.
if $monitoring_enabled {
+ # Prometheus exporters
+ include ::profile::hadoop::monitoring::namenode
+ $namenode_jmx_exporter_config_file =
$::profile::hadoop::monitoring::namenode::jmx_exporter_config_file
+ $prometheus_jmx_exporter_namenode_port =
$::profile::hadoop::monitoring::datanode::prometheus_jmx_exporter_namenode_port
+
+ include ::profile::hadoop::monitoring::resourcemanager
+ $resourcemanager_jmx_exporter_config_file =
$::profile::hadoop::monitoring::resourcemanager::jmx_exporter_config_file
+ $prometheus_jmx_exporter_resourcemanager_port =
$::profile::hadoop::monitoring::resourcemanager::prometheus_jmx_exporter_resourcemanager_port
+
# Icinga process alert for Stand By NameNode
nrpe::monitor_service { 'hadoop-hdfs-namenode':
description => 'Hadoop Namenode - Stand By',
diff --git a/modules/profile/manifests/hadoop/monitoring/history.pp
b/modules/profile/manifests/hadoop/monitoring/history.pp
new file mode 100644
index 0000000..76cfd03
--- /dev/null
+++ b/modules/profile/manifests/hadoop/monitoring/history.pp
@@ -0,0 +1,18 @@
+# Class: profile::hadoop::monitoring::history
+#
+# Sets up Prometheus based monitoring for the Hadoop MapReduce History server
+#
+class profile::hadoop::monitoring::history(
+ $prometheus_nodes = hiera('prometheus_nodes'),
+) {
+ $jmx_exporter_config_file =
'/etc/hadoop/prometheus_mapreduce_history_jmx_exporter.yaml'
+ $prometheus_jmx_exporter_history_port = 10086
+ $java_opts =
"-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=${::ipaddress}:${prometheus_jmx_exporter_history_port}:${jmx_exporter_config_file}"
+ profile::prometheus::jmx_exporter { "mapreduce_history_${::hostname}":
+ hostname => $::hostname,
+ port => $prometheus_jmx_exporter_history_port,
+ prometheus_nodes => $prometheus_nodes,
+ config_file => $jmx_exporter_config_file,
+ source =>
'puppet:///modules/profile/hadoop/prometheus_mapreduce_history_jmx_exporter.yaml',
+ }
+}
\ No newline at end of file
diff --git a/modules/profile/manifests/hadoop/monitoring/namenode.pp
b/modules/profile/manifests/hadoop/monitoring/namenode.pp
new file mode 100644
index 0000000..7497ad3
--- /dev/null
+++ b/modules/profile/manifests/hadoop/monitoring/namenode.pp
@@ -0,0 +1,18 @@
+# Class: profile::hadoop::monitoring::namenode
+#
+# Sets up Prometheus based monitoring for the Hadoop HDFS Namenode
+#
+class profile::hadoop::monitoring::namenode(
+ $prometheus_nodes = hiera('prometheus_nodes'),
+) {
+ $jmx_exporter_config_file =
'/etc/hadoop/prometheus_hdfs_namenode_jmx_exporter.yaml'
+ $prometheus_jmx_exporter_namenode_port = 10080
+ $java_opts =
"-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=${::ipaddress}:${prometheus_jmx_exporter_namenode_port}:${jmx_exporter_config_file}"
+ profile::prometheus::jmx_exporter { "hdfs_namenode_${::hostname}":
+ hostname => $::hostname,
+ port => $prometheus_jmx_exporter_namenode_port,
+ prometheus_nodes => $prometheus_nodes,
+ config_file => $jmx_exporter_config_file,
+ source =>
'puppet:///modules/profile/hadoop/prometheus_hdfs_namenode_jmx_exporter.yaml',
+ }
+}
\ No newline at end of file
diff --git a/modules/profile/manifests/hadoop/monitoring/resourcemanager.pp
b/modules/profile/manifests/hadoop/monitoring/resourcemanager.pp
new file mode 100644
index 0000000..1daad96
--- /dev/null
+++ b/modules/profile/manifests/hadoop/monitoring/resourcemanager.pp
@@ -0,0 +1,18 @@
+# Class: profile::hadoop::monitoring::resourcemanager
+#
+# Sets up Prometheus based monitoring for the Hadoop Yarn Resourcemanager
+#
+class profile::hadoop::monitoring::resourcemanager(
+ $prometheus_nodes = hiera('prometheus_nodes'),
+) {
+ $jmx_exporter_config_file =
'/etc/hadoop/prometheus_yarn_resourcemanager_jmx_exporter.yaml'
+ $prometheus_jmx_exporter_resourcemanager_port = 10083
+ $java_opts =
"-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=${::ipaddress}:${prometheus_jmx_exporter_resourcemanager_port}:${jmx_exporter_config_file}"
+ profile::prometheus::jmx_exporter { "hdfs_datanode_${::hostname}":
+ hostname => $::hostname,
+ port => $prometheus_jmx_exporter_resourcemanager_port,
+ prometheus_nodes => $prometheus_nodes,
+ config_file => $jmx_exporter_config_file,
+ source =>
'puppet:///modules/profile/hadoop/prometheus_yarn_resourcemanager_jmx_exporter.yaml',
+ }
+}
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/395054
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie79831b65b8f03f6f77e495614c2276614f7dd97
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Elukey <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
