ArielGlenn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/391222 )
Change subject: move hardcoded paths out of dump rsync server manifests
......................................................................
move hardcoded paths out of dump rsync server manifests
The new labstore boxes have different mount points and paths,
so we parameterize those.
Bug: T171541
Change-Id: Iae6e6dbaed47c1d210f6ca751a58ab27ed3c9817
---
M modules/dumps/manifests/rsync/media.pp
M modules/dumps/manifests/rsync/pagecounts_ez.pp
M modules/dumps/manifests/rsync/peers.pp
M modules/dumps/manifests/rsync/phab_dump.pp
M modules/dumps/manifests/rsync/public.pp
M modules/dumps/manifests/rsync/slowparse_logs.pp
M modules/dumps/templates/rsync/rsyncd.conf.datasets_to_peers.erb
M modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
M modules/dumps/templates/rsync/rsyncd.conf.media.erb
M modules/dumps/templates/rsync/rsyncd.conf.pagecounts_ez.erb
M modules/dumps/templates/rsync/rsyncd.conf.phab_dump.erb
M modules/dumps/templates/rsync/rsyncd.conf.slowparse_logs.erb
M modules/profile/manifests/dumps/rsyncer.pp
M modules/profile/manifests/dumps/rsyncer_peer.pp
14 files changed, 28 insertions(+), 9 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/dumps/manifests/rsync/media.pp
b/modules/dumps/manifests/rsync/media.pp
index f5f5809..48f030c 100644
--- a/modules/dumps/manifests/rsync/media.pp
+++ b/modules/dumps/manifests/rsync/media.pp
@@ -2,6 +2,7 @@
$hosts_allow = undef,
$user = undef,
$deploygroup = undef,
+ $otherdir = undef,
) {
file { '/etc/rsyncd.d/30-rsync-media.conf':
ensure => 'present',
diff --git a/modules/dumps/manifests/rsync/pagecounts_ez.pp
b/modules/dumps/manifests/rsync/pagecounts_ez.pp
index 9687b0a..55d2113 100644
--- a/modules/dumps/manifests/rsync/pagecounts_ez.pp
+++ b/modules/dumps/manifests/rsync/pagecounts_ez.pp
@@ -2,6 +2,7 @@
$hosts_allow = undef,
$user = undef,
$deploygroup = undef,
+ $otherdir = undef,
) {
file { '/etc/rsyncd.d/30-rsync-pagecounts_ez.conf':
ensure => 'present',
diff --git a/modules/dumps/manifests/rsync/peers.pp
b/modules/dumps/manifests/rsync/peers.pp
index 09a672c..6681272 100644
--- a/modules/dumps/manifests/rsync/peers.pp
+++ b/modules/dumps/manifests/rsync/peers.pp
@@ -1,5 +1,6 @@
class dumps::rsync::peers(
$hosts_allow = undef,
+ $datapath = undef,
) {
file { '/etc/rsyncd.d/10-rsync-datasets_to_peers.conf':
ensure => 'present',
diff --git a/modules/dumps/manifests/rsync/phab_dump.pp
b/modules/dumps/manifests/rsync/phab_dump.pp
index 636cc4e..467026b 100644
--- a/modules/dumps/manifests/rsync/phab_dump.pp
+++ b/modules/dumps/manifests/rsync/phab_dump.pp
@@ -1,5 +1,6 @@
class dumps::rsync::phab_dump(
$hosts_allow = undef,
+ $otherdir = undef,
) {
file { '/etc/rsyncd.d/40-rsync-phab_dump.conf':
ensure => 'present',
diff --git a/modules/dumps/manifests/rsync/public.pp
b/modules/dumps/manifests/rsync/public.pp
index e5c7a0f..456a2d4 100644
--- a/modules/dumps/manifests/rsync/public.pp
+++ b/modules/dumps/manifests/rsync/public.pp
@@ -1,6 +1,7 @@
class dumps::rsync::public(
$hosts_allow = undef,
$publicdir = undef,
+ $otherdir = undef,
) {
file { '/etc/rsyncd.d/20-rsync-dumps_to_public.conf':
ensure => 'present',
diff --git a/modules/dumps/manifests/rsync/slowparse_logs.pp
b/modules/dumps/manifests/rsync/slowparse_logs.pp
index b941ee9..07f79ab 100644
--- a/modules/dumps/manifests/rsync/slowparse_logs.pp
+++ b/modules/dumps/manifests/rsync/slowparse_logs.pp
@@ -2,6 +2,7 @@
$hosts_allow = undef,
$user = undef,
$group = undef,
+ $otherdir = undef,
) {
file { '/etc/rsyncd.d/10-rsync-slowparse-logs.conf':
ensure => 'present',
diff --git a/modules/dumps/templates/rsync/rsyncd.conf.datasets_to_peers.erb
b/modules/dumps/templates/rsync/rsyncd.conf.datasets_to_peers.erb
index 7c6faef..0242c0e 100644
--- a/modules/dumps/templates/rsync/rsyncd.conf.datasets_to_peers.erb
+++ b/modules/dumps/templates/rsync/rsyncd.conf.datasets_to_peers.erb
@@ -8,6 +8,6 @@
uid = root
gid = root
read only = false
-path = /data
+path = <%= @datapath %>
exclude = **bad/ **save/ **not/ **temp/ **tmp/ *.inprog
hosts allow = <%= @hosts_allow %>
diff --git a/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
b/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
index 281987d..f627d50 100644
--- a/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
+++ b/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
@@ -84,7 +84,7 @@
[dumpmirrorsother]
read only = true
-path = /data/xmldatadumps/public/other
+path = <%= @otherdir %>
exclude = **tmp/ **temp/ **bad/ **save/ **archive/
hosts allow = <%= @hosts_allow %>
diff --git a/modules/dumps/templates/rsync/rsyncd.conf.media.erb
b/modules/dumps/templates/rsync/rsyncd.conf.media.erb
index 3548750..07fc2d6 100644
--- a/modules/dumps/templates/rsync/rsyncd.conf.media.erb
+++ b/modules/dumps/templates/rsync/rsyncd.conf.media.erb
@@ -9,6 +9,6 @@
uid = <%= @user %>
gid = <%= deploygroup %>
read only = false
-path = /data/xmldatadumps/public/other/media
+path = <%= @otherdir %>/media
hosts allow = <%= @hosts_allow %>
diff --git a/modules/dumps/templates/rsync/rsyncd.conf.pagecounts_ez.erb
b/modules/dumps/templates/rsync/rsyncd.conf.pagecounts_ez.erb
index d288bd1..f0f7db9 100644
--- a/modules/dumps/templates/rsync/rsyncd.conf.pagecounts_ez.erb
+++ b/modules/dumps/templates/rsync/rsyncd.conf.pagecounts_ez.erb
@@ -9,6 +9,6 @@
uid = <%= @user %>
gid = <%= @deploygroup %>
read only = false
-path = /data/xmldatadumps/public/other/pagecounts-ez
+path = <%= @otherdir %>/pagecounts-ez
hosts allow = <%= @hosts_allow %>
diff --git a/modules/dumps/templates/rsync/rsyncd.conf.phab_dump.erb
b/modules/dumps/templates/rsync/rsyncd.conf.phab_dump.erb
index 31431ef..9458e09 100644
--- a/modules/dumps/templates/rsync/rsyncd.conf.phab_dump.erb
+++ b/modules/dumps/templates/rsync/rsyncd.conf.phab_dump.erb
@@ -7,5 +7,5 @@
uid = root
gid = root
read only = false
-path = /data/xmldatadumps/public/other/misc
+path = <%= @otherdir %>/misc
hosts allow = <%= @hosts_allow %>
diff --git a/modules/dumps/templates/rsync/rsyncd.conf.slowparse_logs.erb
b/modules/dumps/templates/rsync/rsyncd.conf.slowparse_logs.erb
index bee425d..b4fdf42 100644
--- a/modules/dumps/templates/rsync/rsyncd.conf.slowparse_logs.erb
+++ b/modules/dumps/templates/rsync/rsyncd.conf.slowparse_logs.erb
@@ -8,5 +8,5 @@
uid = <%= @user %>
gid = <%= @group %>
read only = false
-path = /data/xmldatadumps/public/other/slow-parse
+path = <%= @otherdir %>/slow-parse
hosts allow = <%= @hosts_allow %>
diff --git a/modules/profile/manifests/dumps/rsyncer.pp
b/modules/profile/manifests/dumps/rsyncer.pp
index 24b5dec..aef0d34 100644
--- a/modules/profile/manifests/dumps/rsyncer.pp
+++ b/modules/profile/manifests/dumps/rsyncer.pp
@@ -15,24 +15,34 @@
hosts_allow => $stats_hosts,
user => 'datasets',
deploygroup => 'wikidev',
+ otherdir => '/data/xmldatadumps/public/other',
}
class {'::dumps::rsync::memfix':}
class {'::dumps::rsync::pagecounts_ez':
hosts_allow => $stats_hosts,
user => 'datasets',
deploygroup => 'wikidev',
+ otherdir => '/data/xmldatadumps/public/other',
}
- class {'::dumps::rsync::peers': hosts_allow => $peer_hosts}
- class {'::dumps::rsync::phab_dump': hosts_allow => $phab_hosts}
+ class {'::dumps::rsync::peers':
+ hosts_allow => $peer_hosts,
+ datapath => '/data',
+ }
+ class {'::dumps::rsync::phab_dump':
+ hosts_allow => $phab_hosts,
+ otherdir => '/data/xmldatadumps/public/other',
+ }
$hosts_allow = join(concat($rsync_clients['ipv4']['external'],
$rsync_clients['ipv6']['external']), ' ')
$publicdir = '/data/xmldatadumps/public'
class {'::dumps::rsync::public':
hosts_allow => $hosts_allow,
publicdir => $publicdir,
+ otherdir => '/data/xmldatadumps/public/other',
}
class {'::dumps::rsync::slowparse_logs':
hosts_allow => $mwlog_hosts,
user => 'datasets',
group => 'datasets',
+ otherdir => '/data/xmldatadumps/public/other',
}
}
diff --git a/modules/profile/manifests/dumps/rsyncer_peer.pp
b/modules/profile/manifests/dumps/rsyncer_peer.pp
index af3d239..f197181 100644
--- a/modules/profile/manifests/dumps/rsyncer_peer.pp
+++ b/modules/profile/manifests/dumps/rsyncer_peer.pp
@@ -6,5 +6,8 @@
$peer_hosts = 'dataset1001.wikimedia.org ms1001.wikimedia.org
dumpsdata1001.eqiad.wmnet dumpsdata1002.eqiad.wmnet labstore1006.wikimedia.org'
class {'::dumps::rsync::default':}
class {'::dumps::rsync::memfix':}
- class {'::dumps::rsync::peers': hosts_allow => $peer_hosts}
+ class {'::dumps::rsync::peers':
+ hosts_allow => $peer_hosts,
+ datapath => '/data',
+ }
}
--
To view, visit https://gerrit.wikimedia.org/r/391222
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Iae6e6dbaed47c1d210f6ca751a58ab27ed3c9817
Gerrit-PatchSet: 4
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
