ArielGlenn has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/380721 )
Change subject: Move daaset nfs server manifests to dump module
......................................................................
Move daaset nfs server manifests to dump module
We now use one class for any nfs service, with different profiles
depending on whether the mount goes to the snapshot hosts, stats
hosts. or all of them. Local path of filesystem is now hardcoded
in the profile; it could turn into a hiera value later.
Bug: T175528
Change-Id: Ia2d61583134863679cc723266fe9cf86b0988c59
---
M hieradata/common.yaml
M modules/dataset/manifests/init.pp
D modules/dataset/manifests/nfs.pp
D modules/dumps/manifests/generation/server/nfs.pp
A modules/dumps/manifests/nfs.pp
R modules/dumps/templates/nfs/default-nfs-common.erb
R modules/dumps/templates/nfs/default-nfs-kernel-server.erb
R modules/dumps/templates/nfs/nfs_exports.erb
A modules/profile/manifests/dumps/nfs/all.pp
A modules/profile/manifests/dumps/nfs/generation.pp
A modules/profile/manifests/dumps/nfs/public.pp
D modules/profile/manifests/dumps/nfs_server.pp
M modules/role/manifests/dumps/generation/server.pp
M modules/role/manifests/dumps/web/xmldumps_active.pp
M modules/role/manifests/dumps/web/xmldumps_fallback.pp
15 files changed, 180 insertions(+), 144 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/21/380721/1
diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 1822f98..1e31398 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -297,19 +297,15 @@
- stat1006.eqiad.wmnet
- dataset1001.wikimedia.org
- thorium.eqiad.wmnet
-dataset_clients_snapshots:
- - snapshot1001.eqiad.wmnet
- - snapshot1005.eqiad.wmnet
- - snapshot1006.eqiad.wmnet
- - snapshot1007.eqiad.wmnet
-dataset_clients_other:
- - stat1005.eqiad.wmnet
- - stat1006.eqiad.wmnet
-dumps_clients_snapshots:
- - snapshot1001.eqiad.wmnet
- - snapshot1005.eqiad.wmnet
- - snapshot1006.eqiad.wmnet
- - snapshot1007.eqiad.wmnet
+dumps_nfs_clients:
+ snapshots:
+ - snapshot1001.eqiad.wmnet
+ - snapshot1005.eqiad.wmnet
+ - snapshot1006.eqiad.wmnet
+ - snapshot1007.eqiad.wmnet
+ other:
+ - stat1005.eqiad.wmnet
+ - stat1006.eqiad.wmnet
dumps_web_rsync_server_clients:
ipv4:
- dataset1001.wikimedia.org
diff --git a/modules/dataset/manifests/init.pp
b/modules/dataset/manifests/init.pp
index d2130cc..c82a4e0 100644
--- a/modules/dataset/manifests/init.pp
+++ b/modules/dataset/manifests/init.pp
@@ -1,17 +1,4 @@
class dataset(
- # args:
- # $nfs: true to share data with snapshot hosts via nfs
- $nfs = true,
- ) {
-
include ::dataset::common
require ::dataset::user
-
- if ($nfs) {
- $nfs_enable = true
- }
- else {
- $nfs_enable = false
- }
- class { '::dataset::nfs': enable => $nfs_enable }
}
diff --git a/modules/dataset/manifests/nfs.pp b/modules/dataset/manifests/nfs.pp
deleted file mode 100644
index 4814ad7..0000000
--- a/modules/dataset/manifests/nfs.pp
+++ /dev/null
@@ -1,53 +0,0 @@
-class dataset::nfs($enable=true) {
-
- if ($enable) {
- $service_ensure = 'running'
- $role_ensure = 'present'
- }
- else {
- $service_ensure = 'stopped'
- $role_ensure = 'absent'
- }
-
- $dataset_clients_snapshots = hiera('dataset_clients_snapshots')
- $dataset_clients_other = hiera('dataset_clients_other')
-
- file { '/etc/exports':
- mode => '0444',
- owner => 'root',
- group => 'root',
- content => template('dataset/nfs_exports.erb'),
- require => Package['nfs-kernel-server'],
- }
-
- require_package('nfs-kernel-server', 'nfs-common', 'rpcbind')
-
- service { 'nfs-kernel-server':
- ensure => $service_ensure,
- require => [
- Package['nfs-kernel-server'],
- File['/etc/exports'],
- ],
- subscribe => File['/etc/exports'],
- }
-
- file { '/etc/default/nfs-common':
- mode => '0444',
- owner => 'root',
- group => 'root',
- source => 'puppet:///modules/dataset/default-nfs-common',
- require => Package['nfs-kernel-server'],
- }
-
- file { '/etc/default/nfs-kernel-server':
- mode => '0444',
- owner => 'root',
- group => 'root',
- source => 'puppet:///modules/dataset/default-nfs-kernel-server',
- require => Package['nfs-kernel-server'],
- }
-
- kmod::options { 'lockd':
- options => 'nlm_udpport=32768 nlm_tcpport=32769',
- }
-}
diff --git a/modules/dumps/manifests/generation/server/nfs.pp
b/modules/dumps/manifests/generation/server/nfs.pp
deleted file mode 100644
index e1a36e5..0000000
--- a/modules/dumps/manifests/generation/server/nfs.pp
+++ /dev/null
@@ -1,47 +0,0 @@
-class dumps::generation::server::nfs(
- $clients = undef,
- $statd_port = undef,
- $statd_out = undef,
- $lockd_udp = undef,
- $lockd_tcp = undef,
- $mountd_port = undef,
-) {
- file { '/etc/exports':
- mode => '0444',
- owner => 'root',
- group => 'root',
- content => template('dumps/generation/nfs_exports.erb'),
- require => Package['nfs-kernel-server'],
- }
-
- require_package('nfs-kernel-server', 'nfs-common', 'rpcbind')
-
- service { 'nfs-kernel-server':
- ensure => 'running',
- require => [
- Package['nfs-kernel-server'],
- File['/etc/exports'],
- ],
- subscribe => File['/etc/exports'],
- }
-
- file { '/etc/default/nfs-common':
- mode => '0444',
- owner => 'root',
- group => 'root',
- content => template('dumps/generation/default-nfs-common.erb'),
- require => Package['nfs-kernel-server'],
- }
-
- file { '/etc/default/nfs-kernel-server':
- mode => '0444',
- owner => 'root',
- group => 'root',
- content => template('dumps/generation/default-nfs-kernel-server.erb'),
- require => Package['nfs-kernel-server'],
- }
-
- kmod::options { 'lockd':
- options => "nlm_udpport=${lockd_udp} nlm_tcpport=${lockd_tcp}",
- }
-}
diff --git a/modules/dumps/manifests/nfs.pp b/modules/dumps/manifests/nfs.pp
new file mode 100644
index 0000000..f4faad3
--- /dev/null
+++ b/modules/dumps/manifests/nfs.pp
@@ -0,0 +1,97 @@
+class dumps::nfs(
+ $clients = undef,
+ $statd_port = undef,
+ $statd_out = undef,
+ $lockd_udp = undef,
+ $lockd_tcp = undef,
+ $mountd_port = undef,
+ $path = undef,
+) {
+ file { '/etc/exports':
+ mode => '0444',
+ owner => 'root',
+ group => 'root',
+ content => template('dumps/generation/nfs_exports.erb'),
+ require => Package['nfs-kernel-server'],
+ }
+
+ require_package('nfs-kernel-server', 'nfs-common', 'rpcbind')
+
+ service { 'nfs-kernel-server':
+ ensure => 'running',
+ require => [
+ Package['nfs-kernel-server'],
+ File['/etc/exports'],
+ ],
+ subscribe => File['/etc/exports'],
+ }
+
+ file { '/etc/default/nfs-common':
+ mode => '0444',
+ owner => 'root',
+ group => 'root',
+ content => template('dumps/generation/default-nfs-common.erb'),
+ require => Package['nfs-kernel-server'],
+ }
+
+ file { '/etc/default/nfs-kernel-server':
+ mode => '0444',
+ owner => 'root',
+ group => 'root',
+ content => template('dumps/generation/default-nfs-kernel-server.erb'),
+ require => Package['nfs-kernel-server'],
+ }
+
+ kmod::options { 'lockd':
+ options => "nlm_udpport=${lockd_udp} nlm_tcpport=${lockd_tcp}",
+ }
+
+ include ::base::firewall
+
+ ferm::service { 'dumps_nfs':
+ proto => 'tcp',
+ port => '2049',
+ srange => '$PRODUCTION_NETWORKS',
+ }
+
+ ferm::service { 'nfs_rpc_mountd':
+ proto => 'tcp',
+ port => $mountd_port,
+ srange => '$PRODUCTION_NETWORKS',
+ }
+
+ ferm::service { 'nfs_rpc_statd':
+ proto => 'tcp',
+ port => $statd_port,
+ srange => '$PRODUCTION_NETWORKS',
+ }
+
+ ferm::service { 'nfs_portmapper_udp':
+ proto => 'udp',
+ port => $portmapper_port,
+ srange => '$PRODUCTION_NETWORKS',
+ }
+
+ ferm::service { 'nfs_portmapper_tcp':
+ proto => 'tcp',
+ port => $portmapper_port,
+ srange => '$PRODUCTION_NETWORKS',
+ }
+
+ ferm::service { 'nfs_lockd_udp':
+ proto => 'udp',
+ port => $lockd_udp,
+ srange => '$PRODUCTION_NETWORKS',
+ }
+
+ ferm::service { 'nfs_lockd_tcp':
+ proto => 'tcp',
+ port => $lockd_tcp,
+ srange => '$PRODUCTION_NETWORKS',
+ }
+
+ monitoring::service { 'nfs':
+ description => 'NFS',
+ check_command => 'check_tcp!2049',
+ }
+}
diff --git a/modules/dumps/templates/generation/default-nfs-common.erb
b/modules/dumps/templates/nfs/default-nfs-common.erb
similarity index 76%
rename from modules/dumps/templates/generation/default-nfs-common.erb
rename to modules/dumps/templates/nfs/default-nfs-common.erb
index b140fd2..d392c91 100644
--- a/modules/dumps/templates/generation/default-nfs-common.erb
+++ b/modules/dumps/templates/nfs/default-nfs-common.erb
@@ -1,7 +1,7 @@
##################################
# THIS FILE IS MANAGED BY PUPPET
#
-# Source: dumps/templates/generation/default-nfs-common.erb
+# Source: dumps/templates/nfs/default-nfs-common.erb
##################################
# If you do not set values for the NEED_ options, they will be attempted
@@ -16,7 +16,7 @@
# when you have a port-based firewall. To use a fixed port, set this
# this variable to a statd argument like: "--port 4000 --outgoing-port 4001".
# For more information, see rpc.statd(8) or
http://wiki.debian.org/SecuringNFS
-STATDOPTS="--port <%=
scope.lookupvar('::dumps::generation::server::nfs::statd_port') -%>
--outgoing-port <%=
scope.lookupvar('::dumps::generation::server::nfs::statd_out') -%>"
+STATDOPTS="--port <%= scope.lookupvar('::dumps::nfs::statd_port') -%>
--outgoing-port <%= scope.lookupvar('::dumps::nfs::statd_out') -%>"
# Do you want to start the gssd daemon? It is required for Kerberos mounts.
NEED_GSSD=
diff --git a/modules/dumps/templates/generation/default-nfs-kernel-server.erb
b/modules/dumps/templates/nfs/default-nfs-kernel-server.erb
similarity index 82%
rename from modules/dumps/templates/generation/default-nfs-kernel-server.erb
rename to modules/dumps/templates/nfs/default-nfs-kernel-server.erb
index 2d69862..c739c58 100644
--- a/modules/dumps/templates/generation/default-nfs-kernel-server.erb
+++ b/modules/dumps/templates/nfs/default-nfs-kernel-server.erb
@@ -1,7 +1,7 @@
##################################
# THIS FILE IS MANAGED BY PUPPET
#
-# Source: dumps/templates/generation/default-nfs-kernel-server.erb
+# Source: dumps/templates/nfs/default-nfs-kernel-server.erb
##################################
# Number of servers to start up
@@ -16,7 +16,7 @@
# a fixed port here using the --port option. For more information,
# see rpc.mountd(8) or http://wiki.debian.org/SecuringNFS
# To disable NFSv4 on the server, specify '--no-nfs-version 4' here
-RPCMOUNTDOPTS="--manage-gids -p <%=
scope.lookupvar('::dumps::generation::server::nfs::mountd_port') -%>"
+RPCMOUNTDOPTS="--manage-gids -p <%=
scope.lookupvar('::dumps::nfs::mountd_port') -%>"
# Do you want to start the svcgssd daemon? It is only required for Kerberos
# exports. Valid alternatives are "yes" and "no"; the default is "no".
diff --git a/modules/dumps/templates/generation/nfs_exports.erb
b/modules/dumps/templates/nfs/nfs_exports.erb
similarity index 79%
rename from modules/dumps/templates/generation/nfs_exports.erb
rename to modules/dumps/templates/nfs/nfs_exports.erb
index 1508ed3..5de2896 100644
--- a/modules/dumps/templates/generation/nfs_exports.erb
+++ b/modules/dumps/templates/nfs/nfs_exports.erb
@@ -12,4 +12,4 @@
# /srv/nfs4/homes gss/krb5i(rw,sync,no_subtree_check)
#
-/data -rw,async,no_root_squash,no_subtree_check <%= Array(@clients).join(' ')
%>
+<%= scope.lookupvar('::dumps::nfs::path') -%>
-rw,async,no_root_squash,no_subtree_check <%= Array(@clients).join(' ') %>
diff --git a/modules/profile/manifests/dumps/nfs/all.pp
b/modules/profile/manifests/dumps/nfs/all.pp
new file mode 100644
index 0000000..a6309a2
--- /dev/null
+++ b/modules/profile/manifests/dumps/nfs/all.pp
@@ -0,0 +1,22 @@
+class profile::dumps::web::nfs::generation(
+ $clients_all = hiera('dumps_nfs_clients'),
+) {
+ $clients = array_concat($clients_all['snapshots'],
$clients_all['other'])
+ $mountd_port = '32767'
+ $statd_port = '32765'
+ $statd_out = '32766'
+ $portmapper_port = '111'
+ $lockd_udp = '32768'
+ $lockd_tcp = '32769'
+ $path = '/data'
+
+ class { '::dumps::nfs':
+ clients => $clients,
+ statd_port => $statd_port,
+ statd_out => $statd_out,
+ lockd_udp => $lockd_udp,
+ lockd_tcp => $lockd_tcp,
+ mountd_port => $mountd_port,
+ path => $path,
+ }
+}
diff --git a/modules/profile/manifests/dumps/nfs/generation.pp
b/modules/profile/manifests/dumps/nfs/generation.pp
new file mode 100644
index 0000000..57b01a7
--- /dev/null
+++ b/modules/profile/manifests/dumps/nfs/generation.pp
@@ -0,0 +1,22 @@
+class profile::dumps::web::nfs::generation(
+ $clients_all = hiera('dumps_nfs_clients'),
+) {
+ $clients = $clients_all['snapshots'],
+ $mountd_port = '32767'
+ $statd_port = '32765'
+ $statd_out = '32766'
+ $portmapper_port = '111'
+ $lockd_udp = '32768'
+ $lockd_tcp = '32769'
+ $path = '/data'
+
+ class { '::dumps::nfs':
+ clients => $clients,
+ statd_port => $statd_port,
+ statd_out => $statd_out,
+ lockd_udp => $lockd_udp,
+ lockd_tcp => $lockd_tcp,
+ mountd_port => $mountd_port,
+ path => $path,
+ }
+}
diff --git a/modules/profile/manifests/dumps/nfs/public.pp
b/modules/profile/manifests/dumps/nfs/public.pp
new file mode 100644
index 0000000..0e7d625
--- /dev/null
+++ b/modules/profile/manifests/dumps/nfs/public.pp
@@ -0,0 +1,22 @@
+class profile::dumps::web::nfs::public(
+ $clients_all = hiera('dumps_nfs_clients'),
+) {
+ $clients = $clients['other']
+ $mountd_port = '32767'
+ $statd_port = '32765'
+ $statd_out = '32766'
+ $portmapper_port = '111'
+ $lockd_udp = '32768'
+ $lockd_tcp = '32769'
+ $path = '/data'
+
+ class { '::dumps::nfs':
+ clients => $clients,
+ statd_port => $statd_port,
+ statd_out => $statd_out,
+ lockd_udp => $lockd_udp,
+ lockd_tcp => $lockd_tcp,
+ mountd_port => $mountd_port,
+ path => $path,
+ }
+}
diff --git a/modules/profile/manifests/dumps/nfs_server.pp
b/modules/profile/manifests/dumps/nfs_server.pp
deleted file mode 100644
index 606d904..0000000
--- a/modules/profile/manifests/dumps/nfs_server.pp
+++ /dev/null
@@ -1,12 +0,0 @@
-class profile::dumps::nfs_server {
- monitoring::service { 'nfs':
- description => 'NFS',
- check_command => 'check_tcp!2049',
- }
-
- ferm::service { 'dumps_nfs':
- proto => 'tcp',
- port => '2049',
- srange => '$PRODUCTION_NETWORKS',
- }
-}
diff --git a/modules/role/manifests/dumps/generation/server.pp
b/modules/role/manifests/dumps/generation/server.pp
index 6af93a7..e0efd4c 100644
--- a/modules/role/manifests/dumps/generation/server.pp
+++ b/modules/role/manifests/dumps/generation/server.pp
@@ -3,5 +3,5 @@
include ::standard
include ::profile::dumps::generation::server
- include ::profile::dumps::nfs_server
+ include ::profile::dumps::nfs::generation
}
diff --git a/modules/role/manifests/dumps/web/xmldumps_active.pp
b/modules/role/manifests/dumps/web/xmldumps_active.pp
index 3c667ca..f249e08 100644
--- a/modules/role/manifests/dumps/web/xmldumps_active.pp
+++ b/modules/role/manifests/dumps/web/xmldumps_active.pp
@@ -8,6 +8,7 @@
include ::profile::dumps::nfs_server
include ::profile::dumps::rsyncer
include ::profile::dumps::fetcher
+ include ::profile::dumps::nfs::all
system::role { 'role::dumps::web::xmldumps': description => 'active web,
nfs and rsync server of xml/sql dumps' }
}
diff --git a/modules/role/manifests/dumps/web/xmldumps_fallback.pp
b/modules/role/manifests/dumps/web/xmldumps_fallback.pp
index cb04220..0f0dea3 100644
--- a/modules/role/manifests/dumps/web/xmldumps_fallback.pp
+++ b/modules/role/manifests/dumps/web/xmldumps_fallback.pp
@@ -7,6 +7,7 @@
include ::profile::dumps::web::rsync_server
include ::profile::dumps::nfs_server
include ::profile::dumps::rsyncer_peer
+ include ::profile::dumps::nfs::all
system::role { 'role::dumps::web::xmldumps': description => 'fallback web,
nfs and rsync server of xml/sql dumps' }
}
--
To view, visit https://gerrit.wikimedia.org/r/380721
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia2d61583134863679cc723266fe9cf86b0988c59
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <ar...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
