ArielGlenn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/379517 )
Change subject: Template-ise rsync/public.pp hosts allow
......................................................................
Template-ise rsync/public.pp hosts allow
Any of the public mirrors for dumps is permitted to access any of the
shares, they are all public after all. We simply configure them for the
convenience of the mirrors.
This moves the hostnames out to a profile parameter. IP addresses are
left in the module, as well as all the specific contact info.
more to do.
Change-Id: I4ac3ddde00afc8b921b5b8846a8e657c0f3cae23
---
M hieradata/common.yaml
M modules/dumps/manifests/rsync/public.pp
R modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
M modules/profile/manifests/dumps/rsyncer.pp
M modules/profile/manifests/dumps/web/rsync_server.pp
5 files changed, 52 insertions(+), 42 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 0f52e5e..c1841ec 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -313,30 +313,34 @@
- stat1006.eqiad.wmnet
dumps_web_rsync_server_clients:
ipv4:
- - dataset1001.wikimedia.org
- - mwlog1001.eqiad.wmnet
- - mwlog2001.codfw.wmnet
- - phab1001.eqiad.wmnet
- - ms1001.wikimedia.org
- - stat1005.eqiad.wmnet
- - stat1006.eqiad.wmnet
- - sagres.c3sl.ufpr.br
- - odysseus.fi.muni.cz
- - odysseus.linux.cz
- - mirror.fr.wickedway.nl
+ internal:
+ - dataset1001.wikimedia.org
+ - mwlog1001.eqiad.wmnet
+ - mwlog2001.codfw.wmnet
+ - phab1001.eqiad.wmnet
+ - ms1001.wikimedia.org
+ - stat1005.eqiad.wmnet
+ - stat1006.eqiad.wmnet
+ external:
+ - sagres.c3sl.ufpr.br
+ - odysseus.fi.muni.cz
+ - odysseus.linux.cz
+ - mirror.fr.wickedway.nl
# disabled as of Feb 10 2017, may come back on line later
-# - wikimedia.wansec.com
- - ftpmirror.your.org
- - ec2-174-129-186-231.compute-1.amazonaws.com
- - ftpmirror-ae0-4.us.your.org
- - crcdtn01.crc.nd.edu
- - wmrsync.crc.nd.edu
- - wikimedia.iconic.vi
- - poincare.acc.umu.se
- - wikimedia.bytemark.co.uk
+# - wikimedia.wansec.com
+ - ftpmirror.your.org
+ - ec2-174-129-186-231.compute-1.amazonaws.com
+ - ftpmirror-ae0-4.us.your.org
+ - crcdtn01.crc.nd.edu
+ - wmrsync.crc.nd.edu
+ - wikimedia.iconic.vi
+ - poincare.acc.umu.se
+ - wikimedia.bytemark.co.uk
ipv6:
- - odysseus.ip6.fi.muni.cz
- - poincare.acc.umu.se
+ internal: []
+ external:
+ - odysseus.ip6.fi.muni.cz
+ - poincare.acc.umu.se
# Schemas names that match this regex
# will not be produced to the eventlogging-valid-mixed
diff --git a/modules/dumps/manifests/rsync/public.pp
b/modules/dumps/manifests/rsync/public.pp
index b22a634..ef38d8a 100644
--- a/modules/dumps/manifests/rsync/public.pp
+++ b/modules/dumps/manifests/rsync/public.pp
@@ -1,11 +1,13 @@
-class dumps::rsync::public {
+class dumps::rsync::public(
+ $hosts_allow = undef,
+) {
include ::dumps::rsync::common
file { '/etc/rsyncd.d/20-rsync-dumps_to_public.conf':
- ensure => 'present',
- mode => '0444',
- owner => 'root',
- group => 'root',
- source => 'puppet:///modules/dumps/rsync/rsyncd.conf.dumps_to_public',
- notify => Exec['update-rsyncd.conf'],
+ ensure => 'present',
+ mode => '0444',
+ owner => 'root',
+ group => 'root',
+ content => template('dumps/rsync/rsyncd.conf.dumps_to_public.erb'),
+ notify => Exec['update-rsyncd.conf'],
}
}
diff --git a/modules/dumps/files/rsync/rsyncd.conf.dumps_to_public
b/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
similarity index 75%
rename from modules/dumps/files/rsync/rsyncd.conf.dumps_to_public
rename to modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
index 6157a45..456572c 100644
--- a/modules/dumps/files/rsync/rsyncd.conf.dumps_to_public
+++ b/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
@@ -33,7 +33,7 @@
read only = true
path = /data/xmldatadumps/public
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/
-hosts allow = sagres.c3sl.ufpr.br odysseus.fi.muni.cz odysseus.linux.cz
odysseus.ip6.fi.muni.cz poincare.acc.umu.se wikimedia.bytemark.co.uk
+hosts allow = <%= @hosts_allow %>
# these are the modules to advertise
@@ -44,7 +44,7 @@
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ /* /*/ /*/*/
include from = /data/xmldatadumps/public/rsync-inc-last-5.txt
-hosts allow = sagres.c3sl.ufpr.br odysseus.fi.muni.cz odysseus.linux.cz
odysseus.ip6.fi.muni.cz poincare.acc.umu.se wikimedia.bytemark.co.uk
+hosts allow = <%= @hosts_allow %>
[dumpslastfour]
read only = true
@@ -53,7 +53,7 @@
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ /* /*/ /*/*/
include from = /data/xmldatadumps/public/rsync-inc-last-4.txt
-hosts allow = sagres.c3sl.ufpr.br odysseus.fi.muni.cz odysseus.linux.cz
odysseus.ip6.fi.muni.cz poincare.acc.umu.se wikimedia.bytemark.co.uk
+hosts allow = <%= @hosts_allow %>
[dumpslastthree]
read only = true
@@ -62,7 +62,7 @@
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ /* /*/ /*/*/
include from = /data/xmldatadumps/public/rsync-inc-last-3.txt
-hosts allow = sagres.c3sl.ufpr.br odysseus.fi.muni.cz odysseus.linux.cz
odysseus.ip6.fi.muni.cz poincare.acc.umu.se wikimedia.bytemark.co.uk
+hosts allow = <%= @hosts_allow %>
[dumpslasttwo]
read only = true
@@ -71,7 +71,7 @@
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ /* /*/ /*/*/
include from = /data/xmldatadumps/public/rsync-inc-last-2.txt
-hosts allow = sagres.c3sl.ufpr.br odysseus.fi.muni.cz odysseus.linux.cz
odysseus.ip6.fi.muni.cz poincare.acc.umu.se wikimedia.bytemark.co.uk
+hosts allow = <%= @hosts_allow %>
[dumpslastone]
read only = true
@@ -80,24 +80,25 @@
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ /* /*/ /*/*/
include from = /data/xmldatadumps/public/rsync-inc-last-1.txt
-hosts allow = sagres.c3sl.ufpr.br odysseus.fi.muni.cz odysseus.linux.cz
odysseus.ip6.fi.muni.cz poincare.acc.umu.se wikimedia.bytemark.co.uk
+hosts allow = <%= @hosts_allow %>
[dumpmirrorsother]
read only = true
path = /data/xmldatadumps/public/other
exclude = **tmp/ **temp/ **bad/ **save/ **archive/
-hosts allow = poincare.acc.umu.se
+hosts allow = <%= @hosts_allow %>
[dumpmirrorsalldumps]
read only = true
# this includes only dumps, no archives, no other datasets
path = /data/xmldatadumps/public
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/
-hosts allow = mirror.fr.wickedway.nl
+hosts allow = <%= @hosts_allow %>
[dumpmirrorseverything]
read only = true
# this includes archives, other datasets
path = /data/xmldatadumps/public
exclude = **tmp/ **temp/ **bad/ **save/
-hosts allow = 199.47.196.26 ftpmirror.your.org
ec2-174-129-186-231.compute-1.amazonaws.com 69.31.98.2 crcdtn01.crc.nd.edu
wmrsync.crc.nd.edu 69.28.137.74
+# ip addresses are: 69.31.98.2 your.org, 199.47.196.26 wansecurity.com,
69.28.137.74 iconicindustry.com
+hosts allow = <%= @hosts_allow %> 199.47.196.26 69.31.98.2 69.28.137.74
diff --git a/modules/profile/manifests/dumps/rsyncer.pp
b/modules/profile/manifests/dumps/rsyncer.pp
index ae9acae..cb6641c 100644
--- a/modules/profile/manifests/dumps/rsyncer.pp
+++ b/modules/profile/manifests/dumps/rsyncer.pp
@@ -1,9 +1,12 @@
-class profile::dumps::rsyncer {
+class profile::dumps::rsyncer(
+ $rsync_clients = hiera('dumps_web_rsync_server_clients'),
+) {
class {'::dumps::rsync::default':}
class {'::dumps::rsync::media':}
class {'::dumps::rsync::memfix':}
class {'::dumps::rsync::pagecounts_ez':}
class {'::dumps::rsync::peers':}
class {'::dumps::rsync::phab_dump':}
- class {'::dumps::rsync::public':}
+ $hosts_allow = join(concat($rsync_clients['ipv4']['external'],
$rsync_clients['ipv6']['external']), ' ')
+ class {'::dumps::rsync::public': hosts_allow => $hosts_allow,}
}
diff --git a/modules/profile/manifests/dumps/web/rsync_server.pp
b/modules/profile/manifests/dumps/web/rsync_server.pp
index 1f2ff4d..a808f45 100644
--- a/modules/profile/manifests/dumps/web/rsync_server.pp
+++ b/modules/profile/manifests/dumps/web/rsync_server.pp
@@ -6,8 +6,8 @@
# a AAAA lookup mode for IPv6 addresses, but this equally fails if only
# an IPv4 address is present.
- $rsync_clients_ipv4_ferm = join($rsync_clients['ipv4'], ' ')
- $rsync_clients_ipv6_ferm = join($rsync_clients['ipv6'], ' ')
+ $rsync_clients_ipv4_ferm = join(concat($rsync_clients['ipv4']['internal'],
$rsync_clients['ipv4']['external']), ' ')
+ $rsync_clients_ipv6_ferm = join(concat($rsync_clients['ipv6']['internal'],
$rsync_clients['ipv6']['external']), ' ')
ferm::service {'dumps_rsyncd_ipv4':
port => '873',
--
To view, visit https://gerrit.wikimedia.org/r/379517
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I4ac3ddde00afc8b921b5b8846a8e657c0f3cae23
Gerrit-PatchSet: 10
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Reedy <re...@wikimedia.org>
Gerrit-Reviewer: ArielGlenn <ar...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
