Elukey has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/395054 )
Change subject: role::hadoop::master|standby: add Prometheus JMX exporter config
......................................................................
role::hadoop::master|standby: add Prometheus JMX exporter config
Also fixing a misuse/abuse of hiera precedently tried for
profile::hadoop::worker, that turned out to be too fragile
(even if more DRY).
Bug: T177458
Change-Id: Ie79831b65b8f03f6f77e495614c2276614f7dd97
---
M hieradata/role/common/analytics_cluster/hadoop/master.yaml
M hieradata/role/common/analytics_cluster/hadoop/standby.yaml
M hieradata/role/common/analytics_cluster/hadoop/worker.yaml
A modules/profile/files/hadoop/prometheus_hdfs_namenode_jmx_exporter.yaml
A modules/profile/files/hadoop/prometheus_mapreduce_history_jmx_exporter.yaml
A modules/profile/files/hadoop/prometheus_yarn_resourcemanager_jmx_exporter.yaml
M modules/profile/manifests/hadoop/master.pp
M modules/profile/manifests/hadoop/master/standby.pp
M modules/profile/manifests/hadoop/monitoring/datanode.pp
A modules/profile/manifests/hadoop/monitoring/history.pp
A modules/profile/manifests/hadoop/monitoring/namenode.pp
M modules/profile/manifests/hadoop/monitoring/nodemanager.pp
A modules/profile/manifests/hadoop/monitoring/resourcemanager.pp
M modules/profile/manifests/hadoop/worker.pp
14 files changed, 98 insertions(+), 27 deletions(-)
Approvals:
Elukey: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/role/common/analytics_cluster/hadoop/master.yaml
b/hieradata/role/common/analytics_cluster/hadoop/master.yaml
index c595c4d..77aecdc 100644
--- a/hieradata/role/common/analytics_cluster/hadoop/master.yaml
+++ b/hieradata/role/common/analytics_cluster/hadoop/master.yaml
@@ -17,11 +17,11 @@
# This setting is used to configure the max heap size for both.
# The default is 1000m, we increase it to 2048m in production.
profile::hadoop::common::yarn_heapsize: 2048
-profile::hadoop::common::yarn_resourcemanager_opts: '-Xms2048m'
-profile::hadoop::common::hadoop_namenode_opts: "-Xms6144m -Xmx6144m"
# Used to set up jvm heap size usage thresholds
profile::hadoop::master::namenode_heapsize: 6144
-# MapReduce History server (running on Hadoop master nodes)
-profile::hadoop::common::mapreduce_history_java_opts: '-Xms4096m -Xmx4096m'
\ No newline at end of file
+# Prometheus JMX Exporter config templates.
+profile::hadoop::common::hadoop_namenode_opts: "-Xms6144m -Xmx6144m
-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=%{::ipaddress}:10080:/etc/hadoop/prometheus_hdfs_namenode_jmx_exporter.yaml"
+profile::hadoop::common::yarn_resourcemanager_opts: "-Xms2048m
-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=%{::ipaddress}:10083:/etc/hadoop/prometheus_yarn_resourcemanager_jmx_exporter.yaml"
+profile::hadoop::common::mapreduce_history_java_opts: "-Xms4096m -Xmx4096m
-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=%{::ipaddress}:10086:/etc/hadoop/prometheus_mapreduce_history_jmx_exporter.yaml"
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/hadoop/standby.yaml
b/hieradata/role/common/analytics_cluster/hadoop/standby.yaml
index 654a0e2..5f2c1e1 100644
--- a/hieradata/role/common/analytics_cluster/hadoop/standby.yaml
+++ b/hieradata/role/common/analytics_cluster/hadoop/standby.yaml
@@ -21,11 +21,10 @@
# This setting is used to configure the max heap size for both.
# The default is 1000m, we increase it to 2048m in production.
profile::hadoop::common::yarn_heapsize: 2048
-profile::hadoop::common::yarn_resourcemanager_opts: '-Xms2048m'
-profile::hadoop::common::hadoop_namenode_opts: "-Xms6144m -Xmx6144m"
# Used to set up jvm heap size usage thresholds
profile::hadoop::standby::namenode_heapsize: 6144
-# MapReduce History server (running on Hadoop master nodes)
-profile::hadoop::common::mapreduce_history_java_opts: '-Xms4096m -Xmx4096m'
\ No newline at end of file
+# Prometheus JMX Exporter config templates.
+profile::hadoop::common::hadoop_namenode_opts: "-Xms6144m -Xmx6144m
-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=%{::ipaddress}:10080:/etc/hadoop/prometheus_hdfs_namenode_jmx_exporter.yaml"
+profile::hadoop::common::yarn_resourcemanager_opts: "-Xms2048m
-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=%{::ipaddress}:10083:/etc/hadoop/prometheus_yarn_resourcemanager_jmx_exporter.yaml"
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/hadoop/worker.yaml
b/hieradata/role/common/analytics_cluster/hadoop/worker.yaml
index 6bf94b7..e4c9838 100644
--- a/hieradata/role/common/analytics_cluster/hadoop/worker.yaml
+++ b/hieradata/role/common/analytics_cluster/hadoop/worker.yaml
@@ -18,7 +18,6 @@
profile::hadoop::common::yarn_heapsize: 4096
profile::hadoop::common::hadoop_heapsize: 4096
-# Prometheus JMX Exporter config templates. The missing variables will be
filled
-# in the worker profile before commons.
-profile::hadoop::common::hadoop_datanode_opts: "-Xms4096m
-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=%{::ipaddress}:%{::profile::hadoop::worker::prometheus_jmx_exporter_datanode_port}:%{::profile::hadoop::worker::datanode_jmx_exporter_config_file}"
-profile::hadoop::common::yarn_nodemanager_opts: "-Xms4096m
-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=%{::ipaddress}:%{::profile::hadoop::worker::prometheus_jmx_exporter_nodemanager_port}:%{::profile::hadoop::worker::nodemanager_jmx_exporter_config_file}"
\ No newline at end of file
+# Prometheus JMX Exporter config templates.
+profile::hadoop::common::hadoop_datanode_opts: "-Xms4096m
-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=%{::ipaddress}:51010:/etc/hadoop/prometheus_hdfs_datanode_jmx_exporter.yaml"
+profile::hadoop::common::yarn_nodemanager_opts: "-Xms4096m
-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=%{::ipaddress}:8141:/etc/hadoop/prometheus_yarn_nodemanager_jmx_exporter.yaml"
\ No newline at end of file
diff --git
a/modules/profile/files/hadoop/prometheus_hdfs_namenode_jmx_exporter.yaml
b/modules/profile/files/hadoop/prometheus_hdfs_namenode_jmx_exporter.yaml
new file mode 100644
index 0000000..73530b2
--- /dev/null
+++ b/modules/profile/files/hadoop/prometheus_hdfs_namenode_jmx_exporter.yaml
@@ -0,0 +1,3 @@
+---
+lowercaseOutputLabelNames: true
+lowercaseOutputName: false
\ No newline at end of file
diff --git
a/modules/profile/files/hadoop/prometheus_mapreduce_history_jmx_exporter.yaml
b/modules/profile/files/hadoop/prometheus_mapreduce_history_jmx_exporter.yaml
new file mode 100644
index 0000000..73530b2
--- /dev/null
+++
b/modules/profile/files/hadoop/prometheus_mapreduce_history_jmx_exporter.yaml
@@ -0,0 +1,3 @@
+---
+lowercaseOutputLabelNames: true
+lowercaseOutputName: false
\ No newline at end of file
diff --git
a/modules/profile/files/hadoop/prometheus_yarn_resourcemanager_jmx_exporter.yaml
b/modules/profile/files/hadoop/prometheus_yarn_resourcemanager_jmx_exporter.yaml
new file mode 100644
index 0000000..73530b2
--- /dev/null
+++
b/modules/profile/files/hadoop/prometheus_yarn_resourcemanager_jmx_exporter.yaml
@@ -0,0 +1,3 @@
+---
+lowercaseOutputLabelNames: true
+lowercaseOutputName: false
\ No newline at end of file
diff --git a/modules/profile/manifests/hadoop/master.pp
b/modules/profile/manifests/hadoop/master.pp
index e5d06ea..ea98b79 100644
--- a/modules/profile/manifests/hadoop/master.pp
+++ b/modules/profile/manifests/hadoop/master.pp
@@ -12,6 +12,7 @@
$hadoop_namenode_heapsize =
hiera('profile::hadoop::master::namenode_heapsize'),
$statsd = hiera('statsd'),
){
+
include ::profile::hadoop::common
class { '::cdh::hadoop::master': }
@@ -65,6 +66,11 @@
# Include icinga alerts if production realm.
if $monitoring_enabled {
+ # Prometheus exporters
+ include ::profile::hadoop::monitoring::namenode
+ include ::profile::hadoop::monitoring::resourcemanager
+ include ::profile::hadoop::monitoring::history
+
# Icinga process alerts for NameNode, ResourceManager and HistoryServer
nrpe::monitor_service { 'hadoop-hdfs-namenode':
description => 'Hadoop Namenode - Primary',
diff --git a/modules/profile/manifests/hadoop/master/standby.pp
b/modules/profile/manifests/hadoop/master/standby.pp
index 9d02235..334e1bb 100644
--- a/modules/profile/manifests/hadoop/master/standby.pp
+++ b/modules/profile/manifests/hadoop/master/standby.pp
@@ -10,7 +10,8 @@
$hadoop_namenode_heapsize =
hiera('profile::hadoop::standby::namenode_heapsize'),
$statsd = hiera('statsd'),
) {
- require ::profile::hadoop::common
+
+ include ::profile::hadoop::common
# Ensure that druid user exists on standby namenodes nodes.
class { '::druid::cdh::hadoop::user': }
@@ -67,6 +68,10 @@
# Include icinga alerts if production realm.
if $monitoring_enabled {
+ # Prometheus exporters
+ include ::profile::hadoop::monitoring::namenode
+ include ::profile::hadoop::monitoring::resourcemanager
+
# Java heap space used alerts.
# The goal is to get alarms for long running memory leaks like T153951.
# Only include heap size alerts if heap size is configured.
diff --git a/modules/profile/manifests/hadoop/monitoring/datanode.pp
b/modules/profile/manifests/hadoop/monitoring/datanode.pp
index c3cbfa4..7d39ea3 100644
--- a/modules/profile/manifests/hadoop/monitoring/datanode.pp
+++ b/modules/profile/manifests/hadoop/monitoring/datanode.pp
@@ -1,13 +1,14 @@
# Class: profile::hadoop::monitoring::datanode
#
-# Sets up Prometheus based monitoring for the Hadoop HDFS Datanode
+# Sets up Prometheus based monitoring for the Hadoop HDFS Datanode.
+# This profile takes care of installing the Prometheus exporter and setting
+# up its configuration file, but it does not instruct the target JVM to use it.
#
class profile::hadoop::monitoring::datanode(
$prometheus_nodes = hiera('prometheus_nodes'),
) {
$jmx_exporter_config_file =
'/etc/hadoop/prometheus_hdfs_datanode_jmx_exporter.yaml'
$prometheus_jmx_exporter_datanode_port = 51010
- $java_opts =
"-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=${::ipaddress}:${prometheus_jmx_exporter_datanode_port}:${jmx_exporter_config_file}"
profile::prometheus::jmx_exporter { "hdfs_datanode_${::hostname}":
hostname => $::hostname,
port => $prometheus_jmx_exporter_datanode_port,
diff --git a/modules/profile/manifests/hadoop/monitoring/history.pp
b/modules/profile/manifests/hadoop/monitoring/history.pp
new file mode 100644
index 0000000..49f1e8b
--- /dev/null
+++ b/modules/profile/manifests/hadoop/monitoring/history.pp
@@ -0,0 +1,19 @@
+# Class: profile::hadoop::monitoring::history
+#
+# Sets up Prometheus based monitoring for the Hadoop MapReduce History server.
+# This profile takes care of installing the Prometheus exporter and setting
+# up its configuration file, but it does not instruct the target JVM to use it.
+#
+class profile::hadoop::monitoring::history(
+ $prometheus_nodes = hiera('prometheus_nodes'),
+) {
+ $jmx_exporter_config_file =
'/etc/hadoop/prometheus_mapreduce_history_jmx_exporter.yaml'
+ $prometheus_jmx_exporter_history_port = 10086
+ profile::prometheus::jmx_exporter { "mapreduce_history_${::hostname}":
+ hostname => $::hostname,
+ port => $prometheus_jmx_exporter_history_port,
+ prometheus_nodes => $prometheus_nodes,
+ config_file => $jmx_exporter_config_file,
+ source =>
'puppet:///modules/profile/hadoop/prometheus_mapreduce_history_jmx_exporter.yaml',
+ }
+}
\ No newline at end of file
diff --git a/modules/profile/manifests/hadoop/monitoring/namenode.pp
b/modules/profile/manifests/hadoop/monitoring/namenode.pp
new file mode 100644
index 0000000..05c639b
--- /dev/null
+++ b/modules/profile/manifests/hadoop/monitoring/namenode.pp
@@ -0,0 +1,19 @@
+# Class: profile::hadoop::monitoring::namenode
+#
+# Sets up Prometheus based monitoring for the Hadoop HDFS Namenode.
+# This profile takes care of installing the Prometheus exporter and setting
+# up its configuration file, but it does not instruct the target JVM to use it.
+#
+class profile::hadoop::monitoring::namenode(
+ $prometheus_nodes = hiera('prometheus_nodes'),
+) {
+ $jmx_exporter_config_file =
'/etc/hadoop/prometheus_hdfs_namenode_jmx_exporter.yaml'
+ $prometheus_jmx_exporter_namenode_port = 10080
+ profile::prometheus::jmx_exporter { "hdfs_namenode_${::hostname}":
+ hostname => $::hostname,
+ port => $prometheus_jmx_exporter_namenode_port,
+ prometheus_nodes => $prometheus_nodes,
+ config_file => $jmx_exporter_config_file,
+ source =>
'puppet:///modules/profile/hadoop/prometheus_hdfs_namenode_jmx_exporter.yaml',
+ }
+}
\ No newline at end of file
diff --git a/modules/profile/manifests/hadoop/monitoring/nodemanager.pp
b/modules/profile/manifests/hadoop/monitoring/nodemanager.pp
index c7f0050..58b98bd 100644
--- a/modules/profile/manifests/hadoop/monitoring/nodemanager.pp
+++ b/modules/profile/manifests/hadoop/monitoring/nodemanager.pp
@@ -1,13 +1,14 @@
# Class: profile::hadoop::monitoring::nodemanager
#
-# Sets up Prometheus based monitoring for the Hadoop NodeManager
+# Sets up Prometheus based monitoring for the Hadoop NodeManager.
+# This profile takes care of installing the Prometheus exporter and setting
+# up its configuration file, but it does not instruct the target JVM to use it.
#
class profile::hadoop::monitoring::nodemanager(
$prometheus_nodes = hiera('prometheus_nodes'),
) {
$jmx_exporter_config_file =
'/etc/hadoop/prometheus_yarn_nodemanager_jmx_exporter.yaml'
$prometheus_jmx_exporter_nodemanager_port = 8141
- $java_opts =
"-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=${::ipaddress}:${prometheus_jmx_exporter_nodemanager_port}:${jmx_exporter_config_file}"
profile::prometheus::jmx_exporter { "yarn_nodemanager_${::hostname}":
hostname => $::hostname,
port => $prometheus_jmx_exporter_nodemanager_port,
diff --git a/modules/profile/manifests/hadoop/monitoring/resourcemanager.pp
b/modules/profile/manifests/hadoop/monitoring/resourcemanager.pp
new file mode 100644
index 0000000..1be1f56
--- /dev/null
+++ b/modules/profile/manifests/hadoop/monitoring/resourcemanager.pp
@@ -0,0 +1,19 @@
+# Class: profile::hadoop::monitoring::resourcemanager
+#
+# Sets up Prometheus based monitoring for the Hadoop Yarn Resourcemanager.
+# This profile takes care of installing the Prometheus exporter and setting
+# up its configuration file, but it does not instruct the target JVM to use it.
+#
+class profile::hadoop::monitoring::resourcemanager(
+ $prometheus_nodes = hiera('prometheus_nodes'),
+) {
+ $jmx_exporter_config_file =
'/etc/hadoop/prometheus_yarn_resourcemanager_jmx_exporter.yaml'
+ $prometheus_jmx_exporter_resourcemanager_port = 10083
+ profile::prometheus::jmx_exporter { "hdfs_datanode_${::hostname}":
+ hostname => $::hostname,
+ port => $prometheus_jmx_exporter_resourcemanager_port,
+ prometheus_nodes => $prometheus_nodes,
+ config_file => $jmx_exporter_config_file,
+ source =>
'puppet:///modules/profile/hadoop/prometheus_yarn_resourcemanager_jmx_exporter.yaml',
+ }
+}
\ No newline at end of file
diff --git a/modules/profile/manifests/hadoop/worker.pp
b/modules/profile/manifests/hadoop/worker.pp
index 8378f49..d4dbc53 100644
--- a/modules/profile/manifests/hadoop/worker.pp
+++ b/modules/profile/manifests/hadoop/worker.pp
@@ -12,16 +12,6 @@
$statsd = hiera('statsd'),
) {
- if $monitoring_enabled {
- include ::profile::hadoop::monitoring::datanode
- $datanode_jmx_exporter_config_file =
$::profile::hadoop::monitoring::datanode::jmx_exporter_config_file
- $prometheus_jmx_exporter_datanode_port =
$::profile::hadoop::monitoring::datanode::prometheus_jmx_exporter_datanode_port
-
- include ::profile::hadoop::monitoring::nodemanager
- $nodemanager_jmx_exporter_config_file =
$::profile::hadoop::monitoring::nodemanager::jmx_exporter_config_file
- $prometheus_jmx_exporter_nodemanager_port =
$::profile::hadoop::monitoring::nodemanager::prometheus_jmx_exporter_nodemanager_port
- }
-
include ::profile::hadoop::common
# hive::client is nice to have for jobs launched
@@ -107,6 +97,10 @@
}
if $monitoring_enabled {
+ # Prometheus exporters
+ include ::profile::hadoop::monitoring::datanode
+ include ::profile::hadoop::monitoring::nodemanager
+
# Icinga process alerts for DataNode and NodeManager
nrpe::monitor_service { 'hadoop-hdfs-datanode':
description => 'Hadoop DataNode',
--
To view, visit https://gerrit.wikimedia.org/r/395054
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ie79831b65b8f03f6f77e495614c2276614f7dd97
Gerrit-PatchSet: 14
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Elukey <[email protected]>
Gerrit-Reviewer: Elukey <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
