Giuseppe Lavagetto has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/403388 )
Change subject: puppetdb: refactor to role/profile
......................................................................
puppetdb: refactor to role/profile
* Split the content of the puppetdb role in two profiles: one for the
puppetdb application, one for the corresponding postgresql database,
that can be used separately
* Rework puppetmaster::puppetdb::database, that was a mess.
* Move monitoring to the profile for the database, move db tuning to the
module class itself
* Added type checking to the parameters of puppetmaster::puppetdb::database
Change-Id: I93b02c19398e4fe16818910dfd3ed6f0653aa3a6
---
R hieradata/role/common/puppetmaster/puppetdb.yaml
A modules/profile/manifests/puppetdb.pp
A modules/profile/manifests/puppetdb/database.pp
M modules/puppetmaster/manifests/puppetdb/database.pp
R modules/puppetmaster/templates/puppetdb/tuning.conf.erb
M modules/role/manifests/puppetmaster/puppetdb.pp
6 files changed, 132 insertions(+), 102 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/88/403388/1
diff --git a/hieradata/common/puppetmaster/puppetdb.yaml
b/hieradata/role/common/puppetmaster/puppetdb.yaml
similarity index 63%
rename from hieradata/common/puppetmaster/puppetdb.yaml
rename to hieradata/role/common/puppetmaster/puppetdb.yaml
index 4de1c15..6b56a8a 100644
--- a/hieradata/common/puppetmaster/puppetdb.yaml
+++ b/hieradata/role/common/puppetmaster/puppetdb.yaml
@@ -1,14 +1,14 @@
-puppetmaster::puppetdb::postgres_users:
+profile::puppetdb::postgres_users:
replication@nihal-v4:
user: replication
database: all
- password: "%{::puppetmaster::puppetdb::database::replication_pass}"
+ password: "%{::profile::puppetdb::database::replication_password}"
cidr: 10.192.16.184/32
attrs: REPLICATION
puppetdb@nihal-v4:
user: puppetdb
database: puppetdb
- password: "%{::puppetmaster::puppetdb::database::puppetdb_pass}"
+ password: "%{::profile::puppetdb::database::puppetdb_password}"
cidr: 10.192.16.184/32
puppetmaster::puppetdb::master: nitrogen.eqiad.wmnet
puppetmaster::puppetdb::slaves: [nihal.codfw.wmnet]
diff --git a/modules/profile/manifests/puppetdb.pp
b/modules/profile/manifests/puppetdb.pp
new file mode 100644
index 0000000..b8717af
--- /dev/null
+++ b/modules/profile/manifests/puppetdb.pp
@@ -0,0 +1,26 @@
+class profile::puppetdb(
+ $master = hiera('profile::puppetdb::master'),
+ $puppetmasters = hiera('puppetmaster::servers')
+) {
+ # The JVM heap size has been raised to 6G for T170740
+ class { '::puppetmaster::puppetdb':
+ master => $master,
+ heap_size => '6G',
+ }
+
+ # Only the TLS-terminating nginx proxy will be exposed
+ $puppetmasters_ferm = inline_template('<%=
@puppetmasters.values.flatten(1).map { |p| p[\'worker\'] }.sort.join(\' \')%>')
+
+ ferm::service { 'puppetdb':
+ proto => 'tcp',
+ port => 443,
+ notrack => true,
+ srange => "@resolve((${puppetmasters_ferm}))",
+ }
+
+ ferm::service { 'puppetdb-cumin':
+ proto => 'tcp',
+ port => 443,
+ srange => '$CUMIN_MASTERS',
+ }
+}
diff --git a/modules/profile/manifests/puppetdb/database.pp
b/modules/profile/manifests/puppetdb/database.pp
new file mode 100644
index 0000000..13b5407
--- /dev/null
+++ b/modules/profile/manifests/puppetdb/database.pp
@@ -0,0 +1,52 @@
+# == Class profile::puppetdb::database
+#
+# Sets up a puppetdb postgresql database.
+#
+class profile::puppetdb::database(
+ $master = hiera('profile::puppetdb::master'),
+ $slaves = hiera('profile::puppetdb::slaves'),
+ $shared_buffers = hiera('profile::puppetdb::database::shared_buffers',
'7680MB'),
+ $replication_password = hiera('puppetdb::password::replication'),
+ $puppetdb_password = hiera('puppetdb::password::rw'),
+ $users = hiera('profile::puppetdb::database::users', {}),
+) {
+ include ::passwords::postgres
+
+ $pgversion = $::lsbdistcodename ? {
+ 'stretch' => '9.6',
+ 'jessie' => '9.4',
+ }
+ $slave_range = join($slaves, ' ')
+
+ $role = $master ? {
+ $::fqdn => 'master',
+ default => 'slave',
+ }
+
+ class { '::puppetmaster::puppetdb::database':
+ master => $master,
+ pgversion => $pgversion,
+ replication_pass => $replication_password,
+ puppetdb_pass => $puppetdb_password,
+ puppetdb_users => $users,
+ }
+
+ # Monitoring
+ class { '::prometheus::postgres_exporter': }
+
+ if $role == 'slave' {
+ class { 'postgresql::slave::monitoring':
+ pg_master => $master,
+ pg_user => 'replication',
+ pg_password => $replication_password,
+ }
+ }
+
+ # Firewall rules
+ # Allow connections from all the slaves
+ ferm::service { 'postgresql_puppetdb':
+ proto => 'tcp',
+ port => 5432,
+ srange => "@resolve((${slave_range}))",
+ }
+}
diff --git a/modules/puppetmaster/manifests/puppetdb/database.pp
b/modules/puppetmaster/manifests/puppetdb/database.pp
index ef6b130..96c6270 100644
--- a/modules/puppetmaster/manifests/puppetdb/database.pp
+++ b/modules/puppetmaster/manifests/puppetdb/database.pp
@@ -1,21 +1,53 @@
-# Class puppetmaster::puppetdb::database
+# == Class puppetmaster::puppetdb::database
#
# Sets up the postgresql database
-class puppetmaster::puppetdb::database($master) {
- $replication_pass = hiera('puppetdb::password::replication')
- $puppetdb_pass = hiera('puppetdb::password::rw')
+#
+# === Parameters
+# [*master*] is the master server fqdn
+#
+# [*pgversion*] The postgresql version.
+#
+# [*shared_buffers*] The size of the postgresql shared buffer to use
+#
+# [*replication_pass*] The replication password
+#
+# [*puppetdb_pass*] Password for the puppetdb user,
+#
+# [*puppetdb_users*] Hash of users to create (if any), additionally to the
local ones
+#
+class puppetmaster::puppetdb::database(
+ String $master,
+ Enum['9.4', '9.6'] $pgversion,
+ String $shared_buffers,
+ String $replication_pass,
+ String $puppetdb_pass,
+ Hash $puppetdb_users={},
+) {
+ # Tuning
+ file { "/etc/postgresql/${pgversion}/main/tuning.conf":
+ ensure => 'present',
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ content => template('puppetmaster/puppetdb/tuning.conf.erb'),
+ }
- if $master == $::fqdn {
- # We do this for the require in postgres::db
- $require_class = 'postgresql::master'
+ sysctl::parameters { 'postgres_shmem':
+ values => {
+ # That is derived after tuning postgresql, deriving automatically
is
+ # not the safest idea yet.
+ 'kernel.shmmax' => 8388608000,
+ },
+ }
+
+ $on_master = ($master == $::fqdn)
+ if $on_master {
class { '::postgresql::master':
includes => ['tuning.conf'],
root_dir => '/srv/postgres',
use_ssl => true,
}
- $on_master = true
} else {
- $require_class = 'postgresql::slave'
class { '::postgresql::slave':
includes => ['tuning.conf'],
master_server => $master,
@@ -23,20 +55,13 @@
replication_pass => $replication_pass,
use_ssl => true,
}
- $on_master = false
- }
- class { 'prometheus::postgres_exporter': # lint:ignore:wmf_styleguide
- require => Class[$require_class],
}
# Postgres replication and users
- $postgres_users = hiera('puppetmaster::puppetdb::postgres_users', undef)
- if $postgres_users {
- $postgres_users_defaults = {
- pgversion => 9.4,
- master => $on_master,
+ $puppetdb_users.each |$pg_name, $config| {
+ $actual_config = merge($config, {'master' => $on_master, 'pgversion'
=> $pgversion})
+ postgresql::user { $pg_name:
+ * => $config
}
- create_resources(postgresql::user, $postgres_users,
- $postgres_users_defaults)
}
# Create the puppetdb user for localhost
# This works on every server and is used for read-only db lookups
@@ -46,7 +71,7 @@
database => 'puppetdb',
password => $puppetdb_pass,
cidr => "${::ipaddress}/32",
- pgversion => '9.4',
+ pgversion => $pgversion,
master => $on_master,
}
@@ -60,7 +85,6 @@
# Create the database
postgresql::db { 'puppetdb':
owner => 'puppetdb',
- require => Class[$require_class],
}
exec { 'create_tgrm_extension':
@@ -69,5 +93,4 @@
user => 'postgres',
require => Postgresql::Db['puppetdb'],
}
-
}
diff --git a/modules/role/templates/puppetdb/tuning.conf.erb
b/modules/puppetmaster/templates/puppetdb/tuning.conf.erb
similarity index 100%
rename from modules/role/templates/puppetdb/tuning.conf.erb
rename to modules/puppetmaster/templates/puppetdb/tuning.conf.erb
diff --git a/modules/role/manifests/puppetmaster/puppetdb.pp
b/modules/role/manifests/puppetmaster/puppetdb.pp
index 86637c3..4736a36 100644
--- a/modules/role/manifests/puppetmaster/puppetdb.pp
+++ b/modules/role/manifests/puppetmaster/puppetdb.pp
@@ -1,85 +1,14 @@
# filtertags: labs-project-deployment-prep labs-project-automation-framework
labs-project-toolsbeta
-class role::puppetmaster::puppetdb (
- $shared_buffers = '7680MB'
-) {
+class role::puppetmaster::puppetdb {
include ::standard
- include ::base::firewall
- include ::passwords::postgres
-
- $pgversion = $::lsbdistcodename ? {
- 'stretch' => '9.6',
- 'jessie' => '9.4',
- }
-
- $master = hiera('puppetmaster::puppetdb::master')
- $slaves = hiera('puppetmaster::puppetdb::slaves')
- $slave_range = join($slaves, ' ')
-
- $role = $master ? {
- $::fqdn => 'master',
- default => 'slave',
- }
+ include ::profile::base::firewall
+ include ::profile::puppetdb::database
+ include ::profile::puppetdb
# Monitor the Postgresql replication lag
- if $role == 'slave' {
- $pg_password = hiera('puppetdb::password::replication')
- class { 'postgresql::slave::monitoring':
- pg_master => $master,
- pg_user => 'replication',
- pg_password => $pg_password,
- }
- }
- system::role { "puppetmaster::puppetdb (postgres ${role})":
+ system::role { "puppetmaster::puppetdb (postgres
${::profile::puppetdb::role})":
ensure => 'present',
description => 'PuppetDB server',
- }
-
- ferm::service { 'postgresql_puppetdb':
- proto => 'tcp',
- port => 5432,
- srange => "@resolve((${slave_range}))",
- }
-
- # Only the TLS-terminating nginx proxy will be exposed
- $puppetmasters_ferm = inline_template('<%= scope.call_function(:hiera,
[\'puppetmaster::servers\']).values.flatten(1).map { |p| p[\'worker\']
}.sort.join(\' \')%>')
- ferm::service { 'puppetdb':
- proto => 'tcp',
- port => 443,
- notrack => true,
- srange => "@resolve((${puppetmasters_ferm}))",
- }
-
- ferm::service { 'puppetdb-cumin':
- proto => 'tcp',
- port => 443,
- srange => '$CUMIN_MASTERS',
- }
-
- # Tuning
- file { "/etc/postgresql/${pgversion}/main/tuning.conf":
- ensure => 'present',
- owner => 'root',
- group => 'root',
- mode => '0444',
- content => template('role/puppetdb/tuning.conf.erb'),
- }
-
- sysctl::parameters { 'postgres_shmem':
- values => {
- # That is derived after tuning postgresql, deriving automatically
is
- # not the safest idea yet.
- 'kernel.shmmax' => 8388608000,
- },
- }
-
- class { '::puppetmaster::puppetdb::database':
- master => $master,
- }
-
- # The JVM heap size has been raised to 6G for T170740
- class { '::puppetmaster::puppetdb':
- master => $master,
- heap_size => '6G',
}
}
--
To view, visit https://gerrit.wikimedia.org/r/403388
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I93b02c19398e4fe16818910dfd3ed6f0653aa3a6
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Giuseppe Lavagetto <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
