ArielGlenn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/381793 )
Change subject: move hardcoded path to public dumps dir out of most dumps rsync
manifests
......................................................................
move hardcoded path to public dumps dir out of most dumps rsync manifests
Bug: T175528
Change-Id: I0b84bd4eaf53991d5854c22dac5466b1d0166f5f
---
M modules/dumps/manifests/copying/labs.pp
M modules/dumps/manifests/rsync/public.pp
M modules/dumps/manifests/web/xmldumps.pp
M modules/dumps/templates/copying/labs-rsync-cron.sh.erb
M modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
M modules/dumps/templates/web/xmldumps/nginx.conf.erb
M modules/profile/manifests/dumps/rsyncer.pp
M modules/profile/manifests/dumps/web/xmldumps_active.pp
8 files changed, 33 insertions(+), 22 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/dumps/manifests/copying/labs.pp
b/modules/dumps/manifests/copying/labs.pp
index 320bfb0..44faca9 100644
--- a/modules/dumps/manifests/copying/labs.pp
+++ b/modules/dumps/manifests/copying/labs.pp
@@ -1,5 +1,6 @@
class dumps::copying::labs(
$labhost = undef,
+ $publicdir = undef,
) {
file { '/usr/local/bin/wmfdumpsmirror.py':
ensure => 'present',
@@ -18,7 +19,7 @@
user => 'root',
minute => '50',
hour => '3',
- command => "/usr/local/sbin/labs-rsync-cron.sh ${labhost}",
+ command => "/usr/local/sbin/labs-rsync-cron.sh ${labhost}
${publicdir}",
environment => 'MAILTO=ops-du...@wikimedia.org',
require => File['/usr/local/bin/wmfdumpsmirror.py',
'/usr/local/sbin/labs-rsync-cron.sh'],
diff --git a/modules/dumps/manifests/rsync/public.pp
b/modules/dumps/manifests/rsync/public.pp
index ef38d8a..7f618b7 100644
--- a/modules/dumps/manifests/rsync/public.pp
+++ b/modules/dumps/manifests/rsync/public.pp
@@ -1,5 +1,6 @@
class dumps::rsync::public(
$hosts_allow = undef,
+ $publicdir = undef,
) {
include ::dumps::rsync::common
file { '/etc/rsyncd.d/20-rsync-dumps_to_public.conf':
diff --git a/modules/dumps/manifests/web/xmldumps.pp
b/modules/dumps/manifests/web/xmldumps.pp
index 2f3337a..1a445ed 100644
--- a/modules/dumps/manifests/web/xmldumps.pp
+++ b/modules/dumps/manifests/web/xmldumps.pp
@@ -32,7 +32,7 @@
}
include dumps::web::nginx_logrot
- file { '/data/xmldatadumps/public/favicon.ico':
+ file { "${publicdir}/favicon.ico":
source => 'puppet:///modules/dumps/web/xmldumps/favicon.ico',
owner => 'root',
group => 'root',
diff --git a/modules/dumps/templates/copying/labs-rsync-cron.sh.erb
b/modules/dumps/templates/copying/labs-rsync-cron.sh.erb
index cb975ac..20bacb1 100755
--- a/modules/dumps/templates/copying/labs-rsync-cron.sh.erb
+++ b/modules/dumps/templates/copying/labs-rsync-cron.sh.erb
@@ -17,16 +17,17 @@
fi
}
-if [ -z "$1" ]; then
- echo "Usage: $0 hostname"
+if [ -z "$1" -o -z "$2" ]; then
+ echo "Usage: $0 hostname sourcedir"
exit 1
fi
desthost="$1"
+sourcedir="$2"
running=`pgrep -u root -f "python /usr/local/bin/wmfdumpsmirror.py
--dest_hostname ${desthost}"`
if [ -z "$running" ]; then
- python /usr/local/bin/wmfdumpsmirror.py --dest_hostname ${desthost}
--sourcedir /data/xmldatadumps/public --destdir dumps/public --filesperjob 50
--sizeperjob 5G --workercount 1 --rsynclist
rsync-filelist-last-3-good.txt.rsync --rsyncargs -rlptqgo,--bwlimit=50000
+ python /usr/local/bin/wmfdumpsmirror.py --dest_hostname ${desthost}
--sourcedir $sourcedir --destdir dumps/public --filesperjob 50 --sizeperjob 5G
--workercount 1 --rsynclist rsync-filelist-last-3-good.txt.rsync --rsyncargs
-rlptqgo,--bwlimit=50000
fi
# fixme need to ensure ${desthost}::dumps/public/wikidatawiki/entities/ exists
diff --git a/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
b/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
index 456572c..1a282ea 100644
--- a/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
+++ b/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
@@ -31,7 +31,7 @@
[dumpmirrorslastfivedumps]
# not really, we should transition people to using the module below
read only = true
-path = /data/xmldatadumps/public
+path = <%= @publicdir %>
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/
hosts allow = <%= @hosts_allow %>
@@ -40,46 +40,46 @@
[dumpslastfive]
read only = true
# this includes only the last five good dumps.
-path = /data/xmldatadumps/public
+path = <%= @publicdir %>
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ /* /*/ /*/*/
-include from = /data/xmldatadumps/public/rsync-inc-last-5.txt
+include from = <%= @publicdir %>/rsync-inc-last-5.txt
hosts allow = <%= @hosts_allow %>
[dumpslastfour]
read only = true
# this includes only the last four good dumps.
-path = /data/xmldatadumps/public
+path = <%= @publicdir %>
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ /* /*/ /*/*/
-include from = /data/xmldatadumps/public/rsync-inc-last-4.txt
+include from = <%= @publicdir %>/rsync-inc-last-4.txt
hosts allow = <%= @hosts_allow %>
[dumpslastthree]
read only = true
# this includes only the last three good dumps.
-path = /data/xmldatadumps/public
+path = <%= @publicdir %>
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ /* /*/ /*/*/
-include from = /data/xmldatadumps/public/rsync-inc-last-3.txt
+include from = <%= @publicdir %>/rsync-inc-last-3.txt
hosts allow = <%= @hosts_allow %>
[dumpslasttwo]
read only = true
# this includes only the last two good dumps.
-path = /data/xmldatadumps/public
+path = <%= @publicdir %>
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ /* /*/ /*/*/
-include from = /data/xmldatadumps/public/rsync-inc-last-2.txt
+include from = <%= @publicdir %>/rsync-inc-last-2.txt
hosts allow = <%= @hosts_allow %>
[dumpslastone]
read only = true
# this includes only the last good dump.
-path = /data/xmldatadumps/public
+path = <%= @publicdir %>
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ /* /*/ /*/*/
-include from = /data/xmldatadumps/public/rsync-inc-last-1.txt
+include from = <%= @publicdir %>/rsync-inc-last-1.txt
hosts allow = <%= @hosts_allow %>
[dumpmirrorsother]
@@ -91,14 +91,14 @@
[dumpmirrorsalldumps]
read only = true
# this includes only dumps, no archives, no other datasets
-path = /data/xmldatadumps/public
+path = <%= @publicdir %>
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/
hosts allow = <%= @hosts_allow %>
[dumpmirrorseverything]
read only = true
# this includes archives, other datasets
-path = /data/xmldatadumps/public
+path = <%= @publicdir %>
exclude = **tmp/ **temp/ **bad/ **save/
# ip addresses are: 69.31.98.2 your.org, 199.47.196.26 wansecurity.com,
69.28.137.74 iconicindustry.com
hosts allow = <%= @hosts_allow %> 199.47.196.26 69.31.98.2 69.28.137.74
diff --git a/modules/dumps/templates/web/xmldumps/nginx.conf.erb
b/modules/dumps/templates/web/xmldumps/nginx.conf.erb
index 3202687..dafcbc5 100644
--- a/modules/dumps/templates/web/xmldumps/nginx.conf.erb
+++ b/modules/dumps/templates/web/xmldumps/nginx.conf.erb
@@ -25,7 +25,7 @@
<%= @ssl_settings.join("\n ") %>
- root /data/xmldatadumps/public;
+ root <%= @publicdir %>;
location / {
index index.html index.htm;
diff --git a/modules/profile/manifests/dumps/rsyncer.pp
b/modules/profile/manifests/dumps/rsyncer.pp
index ad24167..4946231 100644
--- a/modules/profile/manifests/dumps/rsyncer.pp
+++ b/modules/profile/manifests/dumps/rsyncer.pp
@@ -12,6 +12,10 @@
class {'::dumps::rsync::peers': hosts_allow => $peer_hosts}
class {'::dumps::rsync::phab_dump': hosts_allow => $phab_hosts}
$hosts_allow = join(concat($rsync_clients['ipv4']['external'],
$rsync_clients['ipv6']['external']), ' ')
- class {'::dumps::rsync::public': hosts_allow => $hosts_allow,}
+ $publicdir = '/data/xmldatadumps/public'
+ class {'::dumps::rsync::public':
+ hosts_allow => $hosts_allow,
+ publicdir => $publicdir,
+ }
class {'::dumps::rsync::slowparse_logs': hosts_allow => $mwlog_hosts}
}
diff --git a/modules/profile/manifests/dumps/web/xmldumps_active.pp
b/modules/profile/manifests/dumps/web/xmldumps_active.pp
index 440d9cf..130ad8e 100644
--- a/modules/profile/manifests/dumps/web/xmldumps_active.pp
+++ b/modules/profile/manifests/dumps/web/xmldumps_active.pp
@@ -1,8 +1,9 @@
class profile::dumps::web::xmldumps_active {
+ $publicdir = '/data/xmldatadumps/public'
class {'::dumps::web::xmldumps_active':
do_acme => hiera('do_acme'),
datadir => '/data/xmldatadumps',
- publicdir => '/data/xmldatadumps/public',
+ publicdir => $publicdir,
otherdir => '/data/xmldatadumps/public/other',
logs_dest =>
'stat1005.eqiad.wmnet::srv/log/webrequest/archive/dumps.wikimedia.org/',
htmldumps_server => 'francium.eqiad.wmnet',
@@ -10,5 +11,8 @@
}
# copy dumps and other datasets to fallback host(s) and to labs
class {'::dumps::copying::peers':}
- class {'::dumps::copying::labs': labhost => 'labstore1003.eqiad.wmnet'}
+ class {'::dumps::copying::labs':
+ labhost => 'labstore1003.eqiad.wmnet',
+ publicdir => $publicdir,
+ }
}
--
To view, visit https://gerrit.wikimedia.org/r/381793
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I0b84bd4eaf53991d5854c22dac5466b1d0166f5f
Gerrit-PatchSet: 3
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <ar...@wikimedia.org>
Gerrit-Reviewer: ArielGlenn <ar...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
