ArielGlenn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/398034 )
Change subject: rename 'otherdir' in the dumps modules
......................................................................
rename 'otherdir' in the dumps modules
On some hosts the path for misc dumps and datasets ends in .../other/
and on some hosts not, and it's a horrible name anyways. Other what?
we leave the paths as they are, but change the var names everywhere
to something with meaning.
Also remove more hardcoded paths in the manifests.
Change-Id: I5472a464c2db2335c6e692ff1053c754889ab18d
---
M modules/dumps/files/web/html/analytics_index.html
M modules/dumps/manifests/copying/labs.pp
M modules/dumps/manifests/generation/server/dirs.pp
M modules/dumps/manifests/rsync/media.pp
M modules/dumps/manifests/rsync/pagecounts_ez.pp
M modules/dumps/manifests/rsync/phab_dump.pp
M modules/dumps/manifests/rsync/public.pp
M modules/dumps/manifests/rsync/slowparse_logs.pp
M modules/dumps/manifests/web/dirs.pp
M modules/dumps/manifests/web/fetches/kiwix.pp
M modules/dumps/manifests/web/fetches/stats.pp
M modules/dumps/manifests/web/fetches/wikitech_dumps.pp
M modules/dumps/manifests/web/html.pp
M modules/dumps/manifests/web/xmldumps.pp
M modules/dumps/manifests/web/xmldumps_active.pp
M modules/dumps/templates/copying/labs-rsync-cron.sh.erb
M modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
M modules/dumps/templates/rsync/rsyncd.conf.media.erb
M modules/dumps/templates/rsync/rsyncd.conf.pagecounts_ez.erb
M modules/dumps/templates/rsync/rsyncd.conf.phab_dump.erb
M modules/dumps/templates/rsync/rsyncd.conf.slowparse_logs.erb
M modules/profile/manifests/dumps/fetcher.pp
M modules/profile/manifests/dumps/generation/server/fallback.pp
M modules/profile/manifests/dumps/generation/server/primary.pp
M modules/profile/manifests/dumps/rsyncer.pp
M modules/profile/manifests/dumps/web/xmldumps_active.pp
M modules/profile/manifests/dumps/web/xmldumps_fallback.pp
27 files changed, 136 insertions(+), 132 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/dumps/files/web/html/analytics_index.html
b/modules/dumps/files/web/html/analytics_index.html
index 2f1f6df..7f9ba98 100644
--- a/modules/dumps/files/web/html/analytics_index.html
+++ b/modules/dumps/files/web/html/analytics_index.html
@@ -43,7 +43,10 @@
</p>
<p>
- <strong>Clickstream</strong>: (referer, resource) pairs
extracted from the request logs of Wikipedia. Please visit the <a
href="https://meta.wikimedia.org/wiki/Research:Wikipedia_clickstream";>Clickstream
mediawiki page</a> for detailed information, and the <a
href="https://figshare.com/articles/Wikipedia_Clickstream/1305770";>Clickstream
figshare page</a> for correctly reference this dataset. Available as:
+ <strong>Clickstream</strong>: (referer, resource) pairs
extracted from the request logs of Wikipedia. Please visit
+ the <a
href="https://meta.wikimedia.org/wiki/Research:Wikipedia_clickstream";>Clickstream
mediawiki page</a> for detailed
+ information, and the <a
href="https://figshare.com/articles/Wikipedia_Clickstream/1305770";>Clickstream
figshare page</a> for
+ correctly reference this dataset. Available as:
<ul>
<li><a href="/other/clickstream">Monthly generated
clickstream for wikipedia in English, Russian, German, Spanish and
Japanese.</a></li>
</ul>
diff --git a/modules/dumps/manifests/copying/labs.pp
b/modules/dumps/manifests/copying/labs.pp
index 738803e..2d7061c 100644
--- a/modules/dumps/manifests/copying/labs.pp
+++ b/modules/dumps/manifests/copying/labs.pp
@@ -1,7 +1,7 @@
class dumps::copying::labs(
$labhost = undef,
$xmldumpsdir = undef,
- $otherdir = undef,
+ $miscdatasetsdir = undef,
) {
file { '/usr/local/bin/wmfdumpsmirror.py':
ensure => 'present',
@@ -20,7 +20,7 @@
user => 'root',
minute => '50',
hour => '3',
- command => "/usr/local/sbin/labs-rsync-cron.sh ${labhost}
${xmldumpsdir} ${otherdir}",
+ command => "/usr/local/sbin/labs-rsync-cron.sh ${labhost}
${xmldumpsdir} ${miscdatasetsdir}",
environment => '[email protected]',
require => File['/usr/local/bin/wmfdumpsmirror.py',
'/usr/local/sbin/labs-rsync-cron.sh'],
diff --git a/modules/dumps/manifests/generation/server/dirs.pp
b/modules/dumps/manifests/generation/server/dirs.pp
index f6608d7..1c29f28 100644
--- a/modules/dumps/manifests/generation/server/dirs.pp
+++ b/modules/dumps/manifests/generation/server/dirs.pp
@@ -1,30 +1,30 @@
class dumps::generation::server::dirs(
- $datadir = undef,
- $xmldumpsdir = undef,
- $tempdir = undef,
- $otherdir = undef,
- $user = undef,
- $group = undef,
+ $datadir = undef,
+ $xmldumpsdir = undef,
+ $tempdir = undef,
+ $miscdatasetsdir = undef,
+ $user = undef,
+ $group = undef,
) {
# Directories where dumps of any type are generated
# This list is not for one-off directories, nor for
# directories with incoming rsyncs of datasets
- $cirrussearchdir = "${otherdir}/cirrussearch"
- $xlationdir = "${otherdir}/contenttranslation"
- $categoriesrdfdir = "${otherdir}/categoriesrdf"
- $globalblocksdir = "${otherdir}/globalblocks"
- $medialistsdir = "${otherdir}/imageinfo"
- $incrsdir = "${otherdir}/incr"
- $mediatitlesdir = "${otherdir}/mediatitles"
- $othermiscdir = "${otherdir}/misc"
- $pagetitlesdir = "${otherdir}/pagetitles"
- $othertestfilesdir = "${otherdir}/testfiles"
- $otherwikibasedir = "${otherdir}/wikibase"
- $otherwikibasewikidatadir = "${otherdir}/wikibase/wikidatawiki"
- $otherwikidatadir = "${otherdir}/wikidata"
+ $cirrussearchdir = "${miscdatasetsdir}/cirrussearch"
+ $xlationdir = "${miscdatasetsdir}/contenttranslation"
+ $categoriesrdfdir = "${miscdatasetsdir}/categoriesrdf"
+ $globalblocksdir = "${miscdatasetsdir}/globalblocks"
+ $medialistsdir = "${miscdatasetsdir}/imageinfo"
+ $incrsdir = "${miscdatasetsdir}/incr"
+ $mediatitlesdir = "${miscdatasetsdir}/mediatitles"
+ $othermiscdir = "${miscdatasetsdir}/misc"
+ $pagetitlesdir = "${miscdatasetsdir}/pagetitles"
+ $othertestfilesdir = "${miscdatasetsdir}/testfiles"
+ $otherwikibasedir = "${miscdatasetsdir}/wikibase"
+ $otherwikibasewikidatadir = "${miscdatasetsdir}/wikibase/wikidatawiki"
+ $otherwikidatadir = "${miscdatasetsdir}/wikidata"
# top level directories for various dumps/datasets
- file { [ $datadir, $xmldumpsdir, $otherdir, $tempdir ]:
+ file { [ $datadir, $xmldumpsdir, $miscdatasetsdir, $tempdir ]:
ensure => 'directory',
mode => '0755',
owner => $user,
diff --git a/modules/dumps/manifests/rsync/media.pp
b/modules/dumps/manifests/rsync/media.pp
index 48f030c..75166da 100644
--- a/modules/dumps/manifests/rsync/media.pp
+++ b/modules/dumps/manifests/rsync/media.pp
@@ -2,7 +2,7 @@
$hosts_allow = undef,
$user = undef,
$deploygroup = undef,
- $otherdir = undef,
+ $miscdatasetsdir = undef,
) {
file { '/etc/rsyncd.d/30-rsync-media.conf':
ensure => 'present',
diff --git a/modules/dumps/manifests/rsync/pagecounts_ez.pp
b/modules/dumps/manifests/rsync/pagecounts_ez.pp
index 55d2113..29dffe9 100644
--- a/modules/dumps/manifests/rsync/pagecounts_ez.pp
+++ b/modules/dumps/manifests/rsync/pagecounts_ez.pp
@@ -2,7 +2,7 @@
$hosts_allow = undef,
$user = undef,
$deploygroup = undef,
- $otherdir = undef,
+ $miscdatasetsdir = undef,
) {
file { '/etc/rsyncd.d/30-rsync-pagecounts_ez.conf':
ensure => 'present',
diff --git a/modules/dumps/manifests/rsync/phab_dump.pp
b/modules/dumps/manifests/rsync/phab_dump.pp
index 467026b..fca01e7 100644
--- a/modules/dumps/manifests/rsync/phab_dump.pp
+++ b/modules/dumps/manifests/rsync/phab_dump.pp
@@ -1,6 +1,6 @@
class dumps::rsync::phab_dump(
$hosts_allow = undef,
- $otherdir = undef,
+ $miscdatasetsdir = undef,
) {
file { '/etc/rsyncd.d/40-rsync-phab_dump.conf':
ensure => 'present',
diff --git a/modules/dumps/manifests/rsync/public.pp
b/modules/dumps/manifests/rsync/public.pp
index daa1723..c49bcb7 100644
--- a/modules/dumps/manifests/rsync/public.pp
+++ b/modules/dumps/manifests/rsync/public.pp
@@ -1,7 +1,7 @@
class dumps::rsync::public(
$hosts_allow = undef,
$xmldumpsdir = undef,
- $otherdir = undef,
+ $miscdatasetsdir = undef,
) {
file { '/etc/rsyncd.d/20-rsync-dumps_to_public.conf':
ensure => 'present',
diff --git a/modules/dumps/manifests/rsync/slowparse_logs.pp
b/modules/dumps/manifests/rsync/slowparse_logs.pp
index 07f79ab..7d50b2b 100644
--- a/modules/dumps/manifests/rsync/slowparse_logs.pp
+++ b/modules/dumps/manifests/rsync/slowparse_logs.pp
@@ -2,7 +2,7 @@
$hosts_allow = undef,
$user = undef,
$group = undef,
- $otherdir = undef,
+ $miscdatasetsdir = undef,
) {
file { '/etc/rsyncd.d/10-rsync-slowparse-logs.conf':
ensure => 'present',
diff --git a/modules/dumps/manifests/web/dirs.pp
b/modules/dumps/manifests/web/dirs.pp
index b65cab7..beb4cac 100644
--- a/modules/dumps/manifests/web/dirs.pp
+++ b/modules/dumps/manifests/web/dirs.pp
@@ -1,24 +1,24 @@
class dumps::web::dirs(
$datadir = undef,
$xmldumpsdir = undef,
- $otherdir = undef,
+ $miscdatasetsdir = undef,
$user = undef,
$group = undef,
) {
# Please note that this is incomplete, but new directories
# should be defined in puppet (here).
- $analyticsdir = "${otherdir}/analytics"
- $othermiscdir = "${otherdir}/misc"
- $othertestfilesdir = "${otherdir}/testfiles"
- $otherdir_wikidata_legacy = "${otherdir}/wikidata"
- $otherdir_wikibase = "${otherdir}/wikibase/"
+ $analyticsdir = "${miscdatasetsdir}/analytics"
+ $othermiscdir = "${miscdatasetsdir}/misc"
+ $othertestfilesdir = "${miscdatasetsdir}/testfiles"
+ $miscdatasetsdir_wikidata_legacy = "${miscdatasetsdir}/wikidata"
+ $miscdatasetsdir_wikibase = "${miscdatasetsdir}/wikibase/"
$relative_wikidatawiki = 'other/wikibase/wikidatawiki'
- $xlationdir = "${otherdir}/contenttranslation"
- $cirrussearchdir = "${otherdir}/cirrussearch"
- $medialistsdir = "${otherdir}/imageinfo"
- $pagetitlesdir = "${otherdir}/pagetitles"
- $mediatitlesdir = "${otherdir}/mediatitles"
- $categoriesrdf = "${otherdir}/categoriesrdf"
+ $xlationdir = "${miscdatasetsdir}/contenttranslation"
+ $cirrussearchdir = "${miscdatasetsdir}/cirrussearch"
+ $medialistsdir = "${miscdatasetsdir}/imageinfo"
+ $pagetitlesdir = "${miscdatasetsdir}/pagetitles"
+ $mediatitlesdir = "${miscdatasetsdir}/mediatitles"
+ $categoriesrdf = "${miscdatasetsdir}/categoriesrdf"
# top level dir
file { $datadir:
@@ -29,7 +29,7 @@
}
# top-level dirs for various dump trees
- file { [ $xmldumpsdir, $otherdir ]:
+ file { [ $xmldumpsdir, $miscdatasetsdir ]:
ensure => 'directory',
mode => '0755',
owner => $user,
@@ -55,8 +55,8 @@
}
# subdirs for wikidata/wikibase weekly dumps
- file { [ $otherdir_wikibase, "${xmldumpsdir}/${relative_wikidatawiki}",
- $otherdir_wikidata_legacy ]:
+ file { [ $miscdatasetsdir_wikibase,
"${xmldumpsdir}/${relative_wikidatawiki}",
+ $miscdatasetsdir_wikidata_legacy ]:
ensure => 'directory',
mode => '0755',
diff --git a/modules/dumps/manifests/web/fetches/kiwix.pp
b/modules/dumps/manifests/web/fetches/kiwix.pp
index 86fdb4c..f9d99e6 100644
--- a/modules/dumps/manifests/web/fetches/kiwix.pp
+++ b/modules/dumps/manifests/web/fetches/kiwix.pp
@@ -2,13 +2,13 @@
$user = undef,
$group = undef,
$xmldumpsdir = undef,
- $otherdir = undef,
+ $miscdatasetsdir = undef,
) {
require_package('rsync')
file { "${xmldumpsdir}/kiwix":
ensure => 'link',
- target => "${otherdir}/kiwix",
+ target => "${miscdatasetsdir}/kiwix",
owner => $user,
group => $group,
mode => '0644',
@@ -24,7 +24,7 @@
cron { 'kiwix-mirror-update':
ensure => 'present',
environment => '[email protected]',
- command => "/bin/bash /usr/local/bin/kiwix-rsync-cron.sh
${otherdir}",
+ command => "/bin/bash /usr/local/bin/kiwix-rsync-cron.sh
${miscdatasetsdir}",
user => $user,
minute => '15',
hour => '*/2',
diff --git a/modules/dumps/manifests/web/fetches/stats.pp
b/modules/dumps/manifests/web/fetches/stats.pp
index ab8bdcf..76ebed5 100644
--- a/modules/dumps/manifests/web/fetches/stats.pp
+++ b/modules/dumps/manifests/web/fetches/stats.pp
@@ -1,12 +1,12 @@
class dumps::web::fetches::stats(
$src = undef,
- $otherdir = undef,
+ $miscdatasetsdir = undef,
$user = undef,
) {
# Copies over the mediacounts files from an rsyncable location.
dumps::web::fetches::job { 'mediacounts':
source => "${src}/mediacounts",
- destination => "${otherdir}/mediacounts",
+ destination => "${miscdatasetsdir}/mediacounts",
minute => '41',
user => $user,
}
@@ -15,7 +15,7 @@
# using the current definition of pageviews, from an rsyncable location.
dumps::web::fetches::job { 'pageview':
source => "${src}/{pageview,projectview}/legacy/hourly",
- destination => "${otherdir}/pageviews",
+ destination => "${miscdatasetsdir}/pageviews",
minute => '51',
user => $user,
}
@@ -24,7 +24,7 @@
# using the last access cookie method, from an rsyncable location.
dumps::web::fetches::job { 'unique_devices':
source => "${src}/unique_devices",
- destination => "${otherdir}/unique_devices",
+ destination => "${miscdatasetsdir}/unique_devices",
minute => '31',
user => $user,
}
@@ -32,7 +32,7 @@
# Copies over clickstream files from an rsyncable location.
dumps::web::fetches::job { 'clickstream':
source => "${src}/clickstream",
- destination => "${otherdir}/clickstream",
+ destination => "${miscdatasetsdir}/clickstream",
hour => '4',
user => $user,
}
diff --git a/modules/dumps/manifests/web/fetches/wikitech_dumps.pp
b/modules/dumps/manifests/web/fetches/wikitech_dumps.pp
index 41df251..f63243d 100644
--- a/modules/dumps/manifests/web/fetches/wikitech_dumps.pp
+++ b/modules/dumps/manifests/web/fetches/wikitech_dumps.pp
@@ -1,8 +1,9 @@
class dumps::web::fetches::wikitech_dumps(
- $url = undef,
+ $url = undef,
+ $miscdatasetsdir = undef,
) {
- $wikitechdir = '/data/xmldatadumps/public/other/wikitech'
+ $wikitechdir = "${miscdatasetsdir}/wikitech"
file { $wikitechdir:
ensure => 'directory',
diff --git a/modules/dumps/manifests/web/html.pp
b/modules/dumps/manifests/web/html.pp
index f38b379..cd05ded 100644
--- a/modules/dumps/manifests/web/html.pp
+++ b/modules/dumps/manifests/web/html.pp
@@ -1,16 +1,16 @@
class dumps::web::html(
$datadir = undef,
$xmldumpsdir = undef,
- $otherdir = undef,
+ $miscdatasetsdir = undef,
$webuser = undef,
$webgroup = undef,
) {
class {'dumps::web::dirs':
- datadir => $datadir,
- xmldumpsdir => $xmldumpsdir,
- otherdir => $otherdir,
- user => $webuser,
- group => $webgroup,
+ datadir => $datadir,
+ xmldumpsdir => $xmldumpsdir,
+ miscdatasetsdir => $miscdatasetsdir,
+ user => $webuser,
+ group => $webgroup,
}
file { "${xmldumpsdir}/dumps.css":
@@ -22,36 +22,36 @@
source => 'puppet:///modules/dumps/web/html/dumps.css',
}
- file { "${otherdir}/index.html":
+ file { "${miscdatasetsdir}/index.html":
ensure => 'present',
- path => "${otherdir}/index.html",
+ path => "${miscdatasetsdir}/index.html",
mode => '0644',
owner => 'root',
group => 'root',
source => 'puppet:///modules/dumps/web/html/other_index.html',
}
- file { "${otherdir}/pagecounts-ez/index.html":
+ file { "${miscdatasetsdir}/pagecounts-ez/index.html":
ensure => 'present',
- path => "${otherdir}/pagecounts-ez/index.html",
+ path => "${miscdatasetsdir}/pagecounts-ez/index.html",
mode => '0644',
owner => 'root',
group => 'root',
source => 'puppet:///modules/dumps/web/html/pagecounts-ez_index.html',
}
- file { "${xmldumpsdir}/other/analytics/index.html":
+ file { "${miscdatasetsdir}/analytics/index.html":
ensure => 'present',
- path => "${xmldumpsdir}/other/analytics/index.html",
+ path => "${miscdatasetsdir}/analytics/index.html",
mode => '0644',
owner => 'root',
group => 'root',
source => 'puppet:///modules/dumps/web/html/analytics_index.html',
}
- file { "${otherdir}/poty/index.html":
+ file { "${miscdatasetsdir}/poty/index.html":
ensure => 'present',
- path => "${otherdir}/poty/index.html",
+ path => "${miscdatasetsdir}/poty/index.html",
mode => '0644',
owner => 'root',
group => 'root',
diff --git a/modules/dumps/manifests/web/xmldumps.pp
b/modules/dumps/manifests/web/xmldumps.pp
index 03587cb..7c7e87e 100644
--- a/modules/dumps/manifests/web/xmldumps.pp
+++ b/modules/dumps/manifests/web/xmldumps.pp
@@ -3,18 +3,18 @@
$do_acme = true,
$datadir = undef,
$xmldumpsdir = undef,
- $otherdir = undef,
+ $miscdatasetsdir = undef,
$htmldumps_server = undef,
$xmldumps_server = undef,
$webuser = undef,
$webgroup = undef,
) {
class {'dumps::web::html':
- datadir => $datadir,
- xmldumpsdir => $xmldumpsdir,
- otherdir => $otherdir,
- webuser => $webuser,
- webgroup => $webgroup,
+ datadir => $datadir,
+ xmldumpsdir => $xmldumpsdir,
+ miscdatasetsdir => $miscdatasetsdir,
+ webuser => $webuser,
+ webgroup => $webgroup,
}
class { '::nginx':
diff --git a/modules/dumps/manifests/web/xmldumps_active.pp
b/modules/dumps/manifests/web/xmldumps_active.pp
index 99a96d7..7476d2e 100644
--- a/modules/dumps/manifests/web/xmldumps_active.pp
+++ b/modules/dumps/manifests/web/xmldumps_active.pp
@@ -2,7 +2,7 @@
$do_acme = true,
$datadir = undef,
$xmldumpsdir = undef,
- $otherdir = undef,
+ $miscdatasetsdir = undef,
$logs_dest = undef,
$htmldumps_server = undef,
$xmldumps_server = undef,
@@ -14,7 +14,7 @@
do_acme => $do_acme,
datadir => $datadir,
xmldumpsdir => $xmldumpsdir,
- otherdir => $otherdir,
+ miscdatasetsdir => $miscdatasetsdir,
htmldumps_server => $htmldumps_server,
xmldumps_server => $xmldumps_server,
webuser => $webuser,
diff --git a/modules/dumps/templates/copying/labs-rsync-cron.sh.erb
b/modules/dumps/templates/copying/labs-rsync-cron.sh.erb
index b207472..9f823d7 100755
--- a/modules/dumps/templates/copying/labs-rsync-cron.sh.erb
+++ b/modules/dumps/templates/copying/labs-rsync-cron.sh.erb
@@ -4,7 +4,6 @@
# puppet:///modules/dumps/copying/labs-rsync-cron.sh.erb
#########################################
-otherdir="/data/xmldatadumps/public/other"
bwlimit="--bwlimit=50000"
do_rsync (){
@@ -12,20 +11,20 @@
destdir=$2
base_srcdir=$3
- running=`pgrep -u root -f -x "/usr/bin/rsync -rlptqgo $bwlimit
${otherdir}/${srcdir} ${desthost}::dumps/${destdir}"`
+ running=`pgrep -u root -f -x "/usr/bin/rsync -rlptqgo $bwlimit
${base_srcdir}/${srcdir} ${desthost}::dumps/${destdir}"`
if [ -z "$running" ]; then
/usr/bin/rsync -rlptqgo "$bwlimit" "${base_srcdir}/${srcdir}"
"${desthost}::dumps/${destdir}"
fi
}
if [ -z "$1" -o -z "$2" -o -z "$3" ]; then
- echo "Usage: $0 hostname sourcedir otherdumpsdir"
+ echo "Usage: $0 hostname sourcedir destdir"
exit 1
fi
desthost="$1"
sourcedir="$2"
-otherdumpsdir="$3"
+destdir="$3"
running=`pgrep -u root -f "python /usr/local/bin/wmfdumpsmirror.py
--dest_hostname ${desthost}"`
if [ -z "$running" ]; then
@@ -35,6 +34,6 @@
# fixme need to ensure ${desthost}::dumps/public/wikidatawiki/entities/ exists
#copy from our dumps "other" directory to the labs host copy of dumps
-do_rsync "incr" "" $otherdumpsdir
-do_rsync "wikibase/wikidatawiki/" "public/wikidatawiki/entities/"
$otherdumpsdir
-do_rsync "pageviews" "" $otherdumpsdir
+do_rsync "incr" "" "$destdir"
+do_rsync "wikibase/wikidatawiki/" "public/wikidatawiki/entities/" "$destdir"
+do_rsync "pageviews" "" "$destdir"
diff --git a/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
b/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
index 373a3ef..4a1d37e 100644
--- a/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
+++ b/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
@@ -84,7 +84,7 @@
[dumpmirrorsother]
read only = true
-path = <%= @otherdir %>
+path = <%= @miscdatasetsdir %>
exclude = **tmp/ **temp/ **bad/ **save/ **archive/
hosts allow = <%= @hosts_allow %>
diff --git a/modules/dumps/templates/rsync/rsyncd.conf.media.erb
b/modules/dumps/templates/rsync/rsyncd.conf.media.erb
index fb194d3..577f49f 100644
--- a/modules/dumps/templates/rsync/rsyncd.conf.media.erb
+++ b/modules/dumps/templates/rsync/rsyncd.conf.media.erb
@@ -9,6 +9,6 @@
uid = <%= @user %>
gid = <%= @deploygroup %>
read only = false
-path = <%= @otherdir %>/media
+path = <%= @miscdatasetsdir %>/media
hosts allow = <%= @hosts_allow %>
diff --git a/modules/dumps/templates/rsync/rsyncd.conf.pagecounts_ez.erb
b/modules/dumps/templates/rsync/rsyncd.conf.pagecounts_ez.erb
index f0f7db9..6f694bb 100644
--- a/modules/dumps/templates/rsync/rsyncd.conf.pagecounts_ez.erb
+++ b/modules/dumps/templates/rsync/rsyncd.conf.pagecounts_ez.erb
@@ -9,6 +9,6 @@
uid = <%= @user %>
gid = <%= @deploygroup %>
read only = false
-path = <%= @otherdir %>/pagecounts-ez
+path = <%= @miscdatasetsdir %>/pagecounts-ez
hosts allow = <%= @hosts_allow %>
diff --git a/modules/dumps/templates/rsync/rsyncd.conf.phab_dump.erb
b/modules/dumps/templates/rsync/rsyncd.conf.phab_dump.erb
index 9458e09..9a06c46 100644
--- a/modules/dumps/templates/rsync/rsyncd.conf.phab_dump.erb
+++ b/modules/dumps/templates/rsync/rsyncd.conf.phab_dump.erb
@@ -7,5 +7,5 @@
uid = root
gid = root
read only = false
-path = <%= @otherdir %>/misc
+path = <%= @miscdatasetsdir %>/misc
hosts allow = <%= @hosts_allow %>
diff --git a/modules/dumps/templates/rsync/rsyncd.conf.slowparse_logs.erb
b/modules/dumps/templates/rsync/rsyncd.conf.slowparse_logs.erb
index b4fdf42..8694805 100644
--- a/modules/dumps/templates/rsync/rsyncd.conf.slowparse_logs.erb
+++ b/modules/dumps/templates/rsync/rsyncd.conf.slowparse_logs.erb
@@ -8,5 +8,5 @@
uid = <%= @user %>
gid = <%= @group %>
read only = false
-path = <%= @otherdir %>/slow-parse
+path = <%= @miscdatasetsdir %>/slow-parse
hosts allow = <%= @hosts_allow %>
diff --git a/modules/profile/manifests/dumps/fetcher.pp
b/modules/profile/manifests/dumps/fetcher.pp
index 1210754..b5e12be 100644
--- a/modules/profile/manifests/dumps/fetcher.pp
+++ b/modules/profile/manifests/dumps/fetcher.pp
@@ -1,19 +1,20 @@
class profile::dumps::fetcher(
$xmldumpsdir = hiera('profile::dumps::xmldumpspublicdir'),
- $otherdir = hiera('profile::dumps::miscdumpsdir'),
+ $miscdatasetsdir = hiera('profile::dumps::miscdumpsdir'),
) {
class {'dumps::web::fetches::kiwix':
- user => 'dumpsgen',
- group => 'dumpsgen',
- xmldumpsdir => $xmldumpsdir,
- otherdir => $otherdir,
+ user => 'dumpsgen',
+ group => 'dumpsgen',
+ xmldumpsdir => $xmldumpsdir,
+ miscdatasetsdir => $miscdatasetsdir,
}
class {'dumps::web::fetches::stats':
- src => 'stat1005.eqiad.wmnet::hdfs-archive',
- otherdir => $otherdir,
- user => 'dumpsgen',
+ src => 'stat1005.eqiad.wmnet::hdfs-archive',
+ miscdatasetsdir => $miscdatasetsdir,
+ user => 'dumpsgen',
}
class {'dumps::web::fetches::wikitech_dumps':
- url => 'https://wikitech.wikimedia.org/dumps/',
+ url => 'https://wikitech.wikimedia.org/dumps/',
+ miscdatasetsdir => $miscdatasetsdir,
}
}
diff --git a/modules/profile/manifests/dumps/generation/server/fallback.pp
b/modules/profile/manifests/dumps/generation/server/fallback.pp
index 644f4cc..7f9b9cf 100644
--- a/modules/profile/manifests/dumps/generation/server/fallback.pp
+++ b/modules/profile/manifests/dumps/generation/server/fallback.pp
@@ -2,11 +2,11 @@
class { '::dumpsuser': }
class { '::dumps::generation::server::dirs':
- datadir => '/data/xmldatadumps',
- xmldumpsdir => '/data/xmldatadumps/public',
- tempdir => '/data/xmldatadumps/temp',
- otherdir => '/data/otherdumps',
- user => $dumpsuser::user,
- group => $dumpsuser::group,
+ datadir => '/data/xmldatadumps',
+ xmldumpsdir => '/data/xmldatadumps/public',
+ tempdir => '/data/xmldatadumps/temp',
+ miscdatasetsdir => '/data/otherdumps',
+ user => $dumpsuser::user,
+ group => $dumpsuser::group,
}
}
diff --git a/modules/profile/manifests/dumps/generation/server/primary.pp
b/modules/profile/manifests/dumps/generation/server/primary.pp
index b6a4951..a6ad506 100644
--- a/modules/profile/manifests/dumps/generation/server/primary.pp
+++ b/modules/profile/manifests/dumps/generation/server/primary.pp
@@ -2,12 +2,12 @@
class { '::dumpsuser': }
class { '::dumps::generation::server::dirs':
- datadir => '/data/xmldatadumps',
- xmldumpsdir => '/data/xmldatadumps/public',
- tempdir => '/data/xmldatadumps/temp',
- otherdir => '/data/otherdumps',
- user => $dumpsuser::user,
- group => $dumpsuser::group,
+ datadir => '/data/xmldatadumps',
+ xmldumpsdir => '/data/xmldatadumps/public',
+ tempdir => '/data/xmldatadumps/temp',
+ miscdatasetsdir => '/data/otherdumps',
+ user => $dumpsuser::user,
+ group => $dumpsuser::group,
}
class { '::dumps::generation::server::rsyncer':
diff --git a/modules/profile/manifests/dumps/rsyncer.pp
b/modules/profile/manifests/dumps/rsyncer.pp
index 61bef5e..159ef01 100644
--- a/modules/profile/manifests/dumps/rsyncer.pp
+++ b/modules/profile/manifests/dumps/rsyncer.pp
@@ -15,7 +15,7 @@
$hosts_allow = join(concat($rsync_clients['ipv4']['external'],
$rsync_clients['ipv6']['external']), ' ')
$xmldumpsdir = "${mntpoint}/xmldatadumps/public"
- $otherdir = "${mntpoint}/xmldatadumps/public/other"
+ $miscdatasetsdir = "${mntpoint}/xmldatadumps/public/other"
class {'::dumps::rsync::common':
user => $user,
@@ -25,19 +25,19 @@
class {'::dumps::rsync::default':}
class {'::dumps::rsync::media':
- hosts_allow => $stats_hosts,
- user => $user,
- deploygroup => $deploygroup,
- otherdir => $otherdir,
+ hosts_allow => $stats_hosts,
+ user => $user,
+ deploygroup => $deploygroup,
+ miscdatasetsdir => $miscdatasetsdir,
}
class {'::vm::higher_min_free_kbytes':}
class {'::dumps::rsync::pagecounts_ez':
- hosts_allow => $stats_hosts,
- user => $user,
- deploygroup => $deploygroup,
- otherdir => $otherdir,
+ hosts_allow => $stats_hosts,
+ user => $user,
+ deploygroup => $deploygroup,
+ miscdatasetsdir => $miscdatasetsdir,
}
class {'::dumps::rsync::peers':
@@ -46,21 +46,21 @@
}
class {'::dumps::rsync::phab_dump':
- hosts_allow => $phab_hosts,
- otherdir => $otherdir,
+ hosts_allow => $phab_hosts,
+ miscdatasetsdir => $miscdatasetsdir,
}
class {'::dumps::rsync::public':
- hosts_allow => $hosts_allow,
- xmldumpsdir => $xmldumpsdir,
- otherdir => $otherdir,
+ hosts_allow => $hosts_allow,
+ xmldumpsdir => $xmldumpsdir,
+ miscdatasetsdir => $miscdatasetsdir,
}
class {'::dumps::rsync::slowparse_logs':
- hosts_allow => $mwlog_hosts,
- user => $user,
- group => $group,
- otherdir => $otherdir,
+ hosts_allow => $mwlog_hosts,
+ user => $user,
+ group => $group,
+ miscdatasetsdir => $miscdatasetsdir,
}
class {'::dumps::web::dumplists':
diff --git a/modules/profile/manifests/dumps/web/xmldumps_active.pp
b/modules/profile/manifests/dumps/web/xmldumps_active.pp
index 82f5f36..6eff666 100644
--- a/modules/profile/manifests/dumps/web/xmldumps_active.pp
+++ b/modules/profile/manifests/dumps/web/xmldumps_active.pp
@@ -3,13 +3,13 @@
) {
class { '::dumpsuser': }
$xmldumpsdir = '/data/xmldatadumps/public'
- $otherdir = '/data/xmldatadumps/public/other'
+ $miscdatasetsdir = '/data/xmldatadumps/public/other'
class {'::dumps::web::xmldumps_active':
do_acme => $do_acme,
datadir => '/data/xmldatadumps',
xmldumpsdir => $xmldumpsdir,
- otherdir => '/data/xmldatadumps/public/other',
+ miscdatasetsdir => '/data/xmldatadumps/public/other',
logs_dest =>
'stat1005.eqiad.wmnet::srv/log/webrequest/archive/dumps.wikimedia.org/',
htmldumps_server => 'francium.eqiad.wmnet',
xmldumps_server => 'dumps.wikimedia.org',
@@ -21,8 +21,8 @@
desthost => 'ms1001.wikimedia.org',
}
class {'::dumps::copying::labs':
- labhost => 'labstore1003.eqiad.wmnet',
- xmldumpsdir => $xmldumpsdir,
- otherdir => $otherdir,
+ labhost => 'labstore1003.eqiad.wmnet',
+ xmldumpsdir => $xmldumpsdir,
+ miscdatasetsdir => $miscdatasetsdir,
}
}
diff --git a/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
b/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
index dedb83f..bb58fa9 100644
--- a/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
+++ b/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
@@ -7,7 +7,7 @@
do_acme => $do_acme,
datadir => '/data/xmldatadumps',
xmldumpsdir => '/data/xmldatadumps/public',
- otherdir => '/data/xmldatadumps/public/other',
+ miscdatasetsdir => '/data/xmldatadumps/public/other',
htmldumps_server => 'francium.eqiad.wmnet',
xmldumps_server => 'dumps.wikimedia.org',
webuser => 'dumpsgen',
--
To view, visit https://gerrit.wikimedia.org/r/398034
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5472a464c2db2335c6e692ff1053c754889ab18d
Gerrit-PatchSet: 8
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
