Giuseppe Lavagetto has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/394966 )
Change subject: role::puppetmaster::puppetdb: add Prometheus monitoring for
puppetdb
......................................................................
role::puppetmaster::puppetdb: add Prometheus monitoring for puppetdb
This change adds only a subset of the Mbeans available since using
the JMX agent's whitelist turned out to be more perfomant.
The puppetdb's jvm options are now configurable via hiera to allow
a more friendly labs deployment.
Change-Id: I58f036e85edb98ef4170580d093c42f0bc8ef786
---
M hieradata/role/common/puppetmaster/puppetdb.yaml
A
modules/profile/files/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml
M modules/profile/manifests/puppetdb.pp
M modules/puppetdb/manifests/app.pp
M modules/puppetdb/templates/puppetdb.service.erb
M modules/puppetmaster/manifests/puppetdb.pp
M modules/role/manifests/puppetmaster/puppetdb.pp
7 files changed, 36 insertions(+), 10 deletions(-)
Approvals:
Giuseppe Lavagetto: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/role/common/puppetmaster/puppetdb.yaml
b/hieradata/role/common/puppetmaster/puppetdb.yaml
index fd8c9a8..b3be4d0 100644
--- a/hieradata/role/common/puppetmaster/puppetdb.yaml
+++ b/hieradata/role/common/puppetmaster/puppetdb.yaml
@@ -10,3 +10,4 @@
cidr: 10.192.16.184/32
profile::puppetdb::master: nitrogen.eqiad.wmnet
profile::puppetdb::slaves: [nihal.codfw.wmnet]
+puppetmaster::puppetdb::jvm_opts: '-Xmx6g'
diff --git
a/modules/profile/files/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml
b/modules/profile/files/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml
new file mode 100644
index 0000000..6ea2bc3
--- /dev/null
+++
b/modules/profile/files/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml
@@ -0,0 +1,8 @@
+---
+lowercaseOutputLabelNames: true
+lowercaseOutputName: false
+whitelistObjectNames:
+ - 'com.puppetlabs.puppetdb.command:type=global,name=*'
+ - 'com.puppetlabs.puppetdb.command:type=replace facts.3,name=*'
+ - 'com.puppetlabs.puppetdb.http.server:type=/v3/commands,name=*'
+ - 'com.puppetlabs.puppetdb.http.server:type=/v3/nodes,name=*'
\ No newline at end of file
diff --git a/modules/profile/manifests/puppetdb.pp
b/modules/profile/manifests/puppetdb.pp
index b8717af..7c9bea8 100644
--- a/modules/profile/manifests/puppetdb.pp
+++ b/modules/profile/manifests/puppetdb.pp
@@ -1,13 +1,32 @@
class profile::puppetdb(
$master = hiera('profile::puppetdb::master'),
- $puppetmasters = hiera('puppetmaster::servers')
+ $puppetmasters = hiera('puppetmaster::servers'),
+ $jvm_opts = hiera('profile::puppetdb::jvm_opts', '-Xmx4G'),
+ $prometheus_nodes = hiera('prometheus_nodes'),
) {
+ # Prometheus JMX agent for the Puppetdb's JVM
+ $jmx_exporter_config_file =
'/etc/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml'
+ $prometheus_jmx_exporter_port = 9400
+ $prometheus_java_opts =
"-javaagent:/usr/share/java/prometheus/jmx_prometheus_javaagent.jar=${::ipaddress}:${prometheus_jmx_exporter_port}:${jmx_exporter_config_file}"
# The JVM heap size has been raised to 6G for T170740
class { '::puppetmaster::puppetdb':
- master => $master,
- heap_size => '6G',
+ master => $master,
+ jvm_opts => "${jvm_opts} ${prometheus_java_opts}",
}
+
+ # Export JMX metrics to prometheus
+ profile::prometheus::jmx_exporter { "puppetdb_${::hostname}":
+ hostname => $::hostname,
+ port => $prometheus_jmx_exporter_port,
+ prometheus_nodes => $prometheus_nodes,
+ config_file => $jmx_exporter_config_file,
+ source =>
'puppet:///modules/profile/puppetmaster/puppetdb/jvm_prometheus_puppetdb_jmx_exporter.yaml',
+ }
+
+
+ # Firewall rules
+
# Only the TLS-terminating nginx proxy will be exposed
$puppetmasters_ferm = inline_template('<%=
@puppetmasters.values.flatten(1).map { |p| p[\'worker\'] }.sort.join(\' \')%>')
diff --git a/modules/puppetdb/manifests/app.pp
b/modules/puppetdb/manifests/app.pp
index a012ee1..1b64d57 100644
--- a/modules/puppetdb/manifests/app.pp
+++ b/modules/puppetdb/manifests/app.pp
@@ -11,7 +11,7 @@
$db_user='puppetdb',
$db_password=undef,
$perform_gc=false,
- $heap_size='4G',
+ $jvm_opts='-Xmx4G',
$bind_ip=undef,
$ssldir=puppet_ssldir(),
$command_processing_threads=16,
diff --git a/modules/puppetdb/templates/puppetdb.service.erb
b/modules/puppetdb/templates/puppetdb.service.erb
index cef26bd..3bcd7ba 100644
--- a/modules/puppetdb/templates/puppetdb.service.erb
+++ b/modules/puppetdb/templates/puppetdb.service.erb
@@ -6,8 +6,8 @@
Group=puppetdb
Environment=CONFIG=/etc/puppetdb/conf.d
ExecStartPre=/bin/bash -c "test -e /var/log/puppetdb/puppetdb-oom.hprof && mv
/var/log/puppetdb/puppetdb-oom.hprof /var/log/puppetdb/puppetdb-oom.hprof.prev
|| exit 0"
-ExecStart=/usr/bin/java -Xmx<%= @heap_size %> -XX:+ExitOnOutOfMemoryError \
--XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=/var/log/puppetdb/puppetdb-oom.hprof
-Djava.security.egd=file:/dev/urandom \
+ExecStart=/usr/bin/java <%= @jvm_opts %> \
+-XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=/var/log/puppetdb/puppetdb-oom.hprof
-Djava.security.egd=file:/dev/urandom \
-cp /usr/share/puppetdb/puppetdb.jar clojure.main -m
com.puppetlabs.puppetdb.core services -c ${CONFIG}
ExecReload=/bin/kill -HUP $MAINPID
Restart=always
diff --git a/modules/puppetmaster/manifests/puppetdb.pp
b/modules/puppetmaster/manifests/puppetdb.pp
index 685bdd4..5f88710 100644
--- a/modules/puppetmaster/manifests/puppetdb.pp
+++ b/modules/puppetmaster/manifests/puppetdb.pp
@@ -5,7 +5,7 @@
$master,
$port = 443,
$jetty_port = 8080,
- $heap_size = '4G',
+ $jvm_opts ='-Xmx4G',
) {
requires_os('debian >= jessie')
@@ -38,6 +38,6 @@
db_ro_host => $::fqdn,
db_password => $puppetdb_pass,
perform_gc => ($master == $::fqdn), # only the master must perform GC
- heap_size => $heap_size,
+ jvm_opts => $jvm_opts,
}
}
diff --git a/modules/role/manifests/puppetmaster/puppetdb.pp
b/modules/role/manifests/puppetmaster/puppetdb.pp
index 71abe47..4f46166 100644
--- a/modules/role/manifests/puppetmaster/puppetdb.pp
+++ b/modules/role/manifests/puppetmaster/puppetdb.pp
@@ -5,8 +5,6 @@
include ::profile::puppetdb::database
include ::profile::puppetdb
- # Monitor the Postgresql replication lag
-
system::role { "puppetmaster::puppetdb (postgres
${::profile::puppetdb::database::role})":
ensure => 'present',
description => 'PuppetDB server',
--
To view, visit https://gerrit.wikimedia.org/r/394966
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I58f036e85edb98ef4170580d093c42f0bc8ef786
Gerrit-PatchSet: 14
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Elukey <[email protected]>
Gerrit-Reviewer: Alexandros Kosiaris <[email protected]>
Gerrit-Reviewer: Elukey <[email protected]>
Gerrit-Reviewer: Filippo Giunchedi <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Herron <[email protected]>
Gerrit-Reviewer: Volans <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
