ArielGlenn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/397806 )
Change subject: clean up all references to a 'public dumps dir' on web/nfs
dumps servers
......................................................................
clean up all references to a 'public dumps dir' on web/nfs dumps servers
These hosts should have no notion of what a public dumps directory
is (as opposed to private files), they should only have a path for
xml dumps and one for misc/other dumps.
Change-Id: Ib3d75f53712c80bb2355a8aa8d0434502ae93eab
---
M modules/dumps/files/generation/rsync-to-peers.sh
M modules/dumps/files/web/list-last-good-dumps.sh
M modules/dumps/files/web/list-last-n-good-dumps.py
M modules/dumps/files/web/unpack-dumpstatusfiles.sh
M modules/dumps/manifests/copying/labs.pp
M modules/dumps/manifests/generation/server/dirs.pp
M modules/dumps/manifests/rsync/public.pp
M modules/dumps/manifests/web/cleanup.pp
M modules/dumps/manifests/web/cleanups/xmldumps.pp
M modules/dumps/manifests/web/dirs.pp
M modules/dumps/manifests/web/fetches/kiwix.pp
M modules/dumps/manifests/web/html.pp
M modules/dumps/manifests/web/xmldumps.pp
M modules/dumps/manifests/web/xmldumps_active.pp
M modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
M modules/dumps/templates/web/xmldumps/nginx.conf.erb
M modules/profile/manifests/dumps/fetcher.pp
M modules/profile/manifests/dumps/generation/server/fallback.pp
M modules/profile/manifests/dumps/generation/server/primary.pp
M modules/profile/manifests/dumps/rsyncer.pp
M modules/profile/manifests/dumps/web/cleanup.pp
M modules/profile/manifests/dumps/web/dumpstatusfiles_sync.pp
M modules/profile/manifests/dumps/web/xmldumps_active.pp
M modules/profile/manifests/dumps/web/xmldumps_fallback.pp
24 files changed, 123 insertions(+), 117 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/dumps/files/generation/rsync-to-peers.sh
b/modules/dumps/files/generation/rsync-to-peers.sh
index d5b8b4e..49abbe9 100644
--- a/modules/dumps/files/generation/rsync-to-peers.sh
+++ b/modules/dumps/files/generation/rsync-to-peers.sh
@@ -24,7 +24,7 @@
Example:
- $0 --xmldumpsdir /data/xmldatadumps \\
+ $0 --xmldumpsdir /data/xmldatadumps/public \\
--xmlremotedirs
dumpsdata1002.eqiad.wmnet::data/xmldatadumps/public/,dumpsdata1003.eqiad.wmnet::data/xmldatadumps/public/
\\
--miscdumpsdir /data/otherdumps \\
--miscremotedirs
dumpsdata1002.eqiad.wmnet::data/otherdumps/,dumpsdata1003.eqiad.wmnet::data/otherdumps/
@@ -34,7 +34,7 @@
make_statusfiles_tarball() {
# make tarball of all xml/sql dumps status and html files
- tarballpath="${xmldumpsdir}/public/dumpstatusfiles.tar"
+ tarballpath="${xmldumpsdir}/dumpstatusfiles.tar"
tarballpathgz="${tarballpath}.gz"
# Only pick up the html/json/txt files from the latest run; even if it's
@@ -43,18 +43,18 @@
# the new run started, unless there are 0 minutes between end of
# one dump run across all wikis and start of the next (in which case
# we are cutting things WAY too close with the runs)
- latestwiki=$( cd "${xmldumpsdir}/public"; ls -td *wik* | head -1 )
+ latestwiki=$( cd "$xmldumpsdir"; ls -td *wik* | head -1 )
rm -f "$tarballpathgz"
# dirname is YYYYMMDD, i.e. 8 digits. ignore all other directories.
- latestrun=$( cd "${xmldumpsdir}/public/${latestwiki}" ; ls -d
[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9] | sort | tail -1 )
+ latestrun=$( cd "${xmldumpsdir}/${latestwiki}" ; ls -d
[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9] | sort | tail -1 )
if [ -n "$latestrun" ]; then
# top-level index files first
- ( cd "$xmldumpsdir"; /bin/tar cfp "$tarballpath" public/*html
public/*json )
- # add per-wiki files next: ( cd /data/xmldatadumps; /usr/bin/find
public/ -maxdepth 3 -regextype sed -regex ".*/20171120/.*\(json\|html\|txt\)" )
- ( cd "$xmldumpsdir"; /usr/bin/find "public/" -maxdepth 3 -regextype
sed -regex ".*/${latestrun}/.*\.\(json\|html\|txt\)" | /usr/bin/xargs -s
1048576 /bin/tar rfp "$tarballpath" )
+ ( cd "$xmldumpsdir"; /bin/tar cfp "$tarballpath" *html *json )
+ # add per-wiki files next: ( cd /data/xmldatadumps/public;
/usr/bin/find . -maxdepth 3 -regextype sed -regex
".*/20171120/.*\(json\|html\|txt\)" )
+ ( cd "$xmldumpsdir"; /usr/bin/find "." -maxdepth 3 -regextype sed
-regex ".*/${latestrun}/.*\.\(json\|html\|txt\)" | /usr/bin/xargs -s 1048576
/bin/tar rfp "$tarballpath" )
# if no files found, there will be no tarball created either
if [ -f "$tarballpath" ]; then
diff --git a/modules/dumps/files/web/list-last-good-dumps.sh
b/modules/dumps/files/web/list-last-good-dumps.sh
index f7ca326..2aa6d51 100644
--- a/modules/dumps/files/web/list-last-good-dumps.sh
+++ b/modules/dumps/files/web/list-last-good-dumps.sh
@@ -15,7 +15,7 @@
--xmldumpsdir path to root of xml/sql dumps tree
-Example: $0 --xmldumpsdir /data/xmldatadumps
+Example: $0 --xmldumpsdir /data/xmldatadumps/public
EOF
exit 1
}
@@ -37,10 +37,8 @@
usage && exit 1
fi
-dumpsdir="${xmldumpsdir}/public/"
-
# generate lists of most recent completed successful dumps for rsync (dirs,
files)
-/usr/bin/python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber
1,2,3,4,5 --dirlisting 'rsync-dirlist-last-%s-good.txt' --rsynclists --relpath
--outputdir "$dumpsdir" --dumpsdir "$dumpsdir"
-/usr/bin/python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber
1,2,3,4,5 --filelisting 'rsync-filelist-last-%s-good.txt' --rsynclists
--relpath --outputdir "$dumpsdir" --toplevel --dumpsdir "$dumpsdir"
+/usr/bin/python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber
1,2,3,4,5 --dirlisting 'rsync-dirlist-last-%s-good.txt' --rsynclists --relpath
--outputdir "${xmldumpsdir}/" --dumpsdir "${xmldumpsdir}/"
+/usr/bin/python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber
1,2,3,4,5 --filelisting 'rsync-filelist-last-%s-good.txt' --rsynclists
--relpath --outputdir "${xmldumpsdir}/" --toplevel --dumpsdir "${xmldumpsdir}/"
# these lists can be used for rsync excl/incl on our side, providing shares
that "just work" for the mirrors
-/usr/bin/python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber
1,2,3,4,5 --rsynclisting 'rsync-inc-last-%s.txt' --relpath --outputdir
"$dumpsdir" --toplevel --dumpsdir "$dumpsdir"
+/usr/bin/python /usr/local/bin/list-last-n-good-dumps.py --dumpsnumber
1,2,3,4,5 --rsynclisting 'rsync-inc-last-%s.txt' --relpath --outputdir
"${xmldumpsdir}/" --toplevel --dumpsdir "${xmldumpsdir}/"
diff --git a/modules/dumps/files/web/list-last-n-good-dumps.py
b/modules/dumps/files/web/list-last-n-good-dumps.py
index 46070e5..7b50990 100755
--- a/modules/dumps/files/web/list-last-n-good-dumps.py
+++ b/modules/dumps/files/web/list-last-n-good-dumps.py
@@ -116,9 +116,8 @@
'filelist': 'file_list_templ',
'rsynclist': 'rsync_incl_templ'}
- def get_abs_pubdirpath(self, name):
- """return full path to the location of public dumps,
- as specified in the config file for the entry 'publicdir'"""
+ def get_abs_xmldumpsdirpath(self, name):
+ """return full path to the location of xml dumps"""
return os.path.join(self.dumpsdir, name)
def get_abs_outdirpath(self, name):
@@ -185,7 +184,7 @@
files from in-progress dumps will only be written to the
rsync include file.
"""
- dir_to_check = self.paths.get_abs_pubdirpath(project)
+ dir_to_check = self.paths.get_abs_xmldumpsdirpath(project)
if not os.path.exists(dir_to_check):
return [], None
@@ -240,7 +239,7 @@
files_wanted = ([os.path.join(dir_name, f) for f in dir_contents
if re.search(extensions, f)])
if self.flags['relative']:
- files_wanted = [self.strip_pubdir(f) for f in files_wanted]
+ files_wanted = [self.strip_xmldumpsdir(f) for f in
files_wanted]
except Exception:
pass
return files_wanted
@@ -261,7 +260,7 @@
files_wanted_shortnames = [entry.rstrip().split()[-1] for entry in
entries]
files_wanted = [os.path.join(dir_name, f) for f in
files_wanted_shortnames]
if self.flags['relative']:
- files_wanted = [self.strip_pubdir(f) for f in files_wanted]
+ files_wanted = [self.strip_xmldumpsdir(f) for f in
files_wanted]
except Exception:
pass
return files_wanted
@@ -314,9 +313,10 @@
if not self.paths.list_wanted(output_type):
return
- dirs = [os.path.join(self.paths.get_abs_pubdirpath(project), dirname)
for dirname in dirs]
+ dirs = [os.path.join(self.paths.get_abs_xmldumpsdirpath(project),
dirname)
+ for dirname in dirs]
if self.flags['relative']:
- dirs = [self.strip_pubdir(dirname) for dirname in dirs]
+ dirs = [self.strip_xmldumpsdir(dirname) for dirname in dirs]
if wildcard:
dirs = [os.path.join(dirname, '*') for dirname in dirs]
@@ -334,7 +334,8 @@
if not self.paths.list_wanted(output_type):
return
- dirs = [os.path.join(self.paths.get_abs_pubdirpath(project), dirname)
for dirname in dirs]
+ dirs = [os.path.join(self.paths.get_abs_xmldumpsdirpath(project),
dirname)
+ for dirname in dirs]
if dirs:
fnames_to_write = []
for dirname in dirs:
@@ -360,7 +361,7 @@
# write list of dump-related files from the latest in-progress
directory, if any,
# to all rsync inclusion lists
fnames_done = self.get_fnames_from_dir(os.path.join(
- self.paths.get_abs_pubdirpath(project), dir_in_progress))
+ self.paths.get_abs_xmldumpsdirpath(project), dir_in_progress))
if fnames_done:
for num in self.dumps_num_list:
output_path = self.paths.get_list_output_path(num,
'rsynclist')
@@ -381,7 +382,7 @@
self.write_filenames(dirs, project)
self.write_rsynclists(dirs, dir_in_progress, project)
- def strip_pubdir(self, line):
+ def strip_xmldumpsdir(self, line):
"""remove the path to the public dumps directory from
the beginning of the suppplied line, if it exists"""
if line.startswith(self.paths.dumpsdir + os.sep):
@@ -393,14 +394,14 @@
to rsync --list-only"""
# to make this work we have to feed it a file with the filenames
- # with the publicdir stripped off the front, if it's there
+ # with the xml dumps dir stripped off the front, if it's there
infd = open(fpath, "r")
outfd = open(fpath + ".relpath", "w")
lines = infd.readlines()
infd.close()
for line in lines:
if not self.flags['relative']:
- outfd.write(self.strip_pubdir(line))
+ outfd.write(self.strip_xmldumpsdir(line))
else:
outfd.write(line)
outfd.close()
@@ -427,7 +428,7 @@
list .html, .json, .css, .txt files in top level dir
test, with "" does this work?
"""
- files_in_dir = [f for f in
os.listdir(self.paths.get_abs_pubdirpath(""))
+ files_in_dir = [f for f in
os.listdir(self.paths.get_abs_xmldumpsdirpath(""))
if f.endswith(".html") or f.endswith(".txt") or
f.endswith(".old")
or f.endswith(".css") or f.endswith(".json")]
return files_in_dir
@@ -495,7 +496,7 @@
Options:
-dumpsdir -- directory under which dumps of all wikis reside. tree should
+dumpsdir -- directory under which public dumps of all wikis reside. tree
should
look like: dumpsdir/wikiname/YYYYMMDD
directories under dumpsdir which have no subdir with name in
the format YYYYMMDD will be silently ignored.
@@ -508,7 +509,7 @@
default value: 5
outputdir -- directory in which to write all file listings
default value: none
-relpath -- generate all lists with paths relative to the public directory
+relpath -- generate all lists with paths relative to the xml dumps
directory
specified, instead of writing out the full path
default value: False
rsynclists -- for each file that is produced, write a second file with the
diff --git a/modules/dumps/files/web/unpack-dumpstatusfiles.sh
b/modules/dumps/files/web/unpack-dumpstatusfiles.sh
index d8450aa..0f2b98d 100644
--- a/modules/dumps/files/web/unpack-dumpstatusfiles.sh
+++ b/modules/dumps/files/web/unpack-dumpstatusfiles.sh
@@ -21,7 +21,7 @@
--xmldumpsdir path to root of xml/sql dumps tree for unpacking tarball
--newer file must be newer than this many minutes ago to unpack it
-Example: $0 --xmldumpsdir /data/xmldatadumps --newer 10
+Example: $0 --xmldumpsdir /data/xmldatadumps/public --newer 10
EOF
exit 1
}
@@ -50,7 +50,7 @@
usage && exit 1
fi
-tarballpath="${xmldumpsdir}/public/dumpstatusfiles.tar.gz"
+tarballpath="${xmldumpsdir}/dumpstatusfiles.tar.gz"
if [ ! -e "$tarballpath" ]; then
# no file so do no check
exit 0
diff --git a/modules/dumps/manifests/copying/labs.pp
b/modules/dumps/manifests/copying/labs.pp
index 5dce210..738803e 100644
--- a/modules/dumps/manifests/copying/labs.pp
+++ b/modules/dumps/manifests/copying/labs.pp
@@ -1,6 +1,6 @@
class dumps::copying::labs(
$labhost = undef,
- $publicdir = undef,
+ $xmldumpsdir = undef,
$otherdir = undef,
) {
file { '/usr/local/bin/wmfdumpsmirror.py':
@@ -20,7 +20,7 @@
user => 'root',
minute => '50',
hour => '3',
- command => "/usr/local/sbin/labs-rsync-cron.sh ${labhost}
${publicdir} ${otherdir}",
+ command => "/usr/local/sbin/labs-rsync-cron.sh ${labhost}
${xmldumpsdir} ${otherdir}",
environment => '[email protected]',
require => File['/usr/local/bin/wmfdumpsmirror.py',
'/usr/local/sbin/labs-rsync-cron.sh'],
diff --git a/modules/dumps/manifests/generation/server/dirs.pp
b/modules/dumps/manifests/generation/server/dirs.pp
index 5faaf1e..f6608d7 100644
--- a/modules/dumps/manifests/generation/server/dirs.pp
+++ b/modules/dumps/manifests/generation/server/dirs.pp
@@ -1,14 +1,14 @@
class dumps::generation::server::dirs(
- $user = undef,
- $group = undef,
+ $datadir = undef,
+ $xmldumpsdir = undef,
+ $tempdir = undef,
+ $otherdir = undef,
+ $user = undef,
+ $group = undef,
) {
# Directories where dumps of any type are generated
# This list is not for one-off directories, nor for
# directories with incoming rsyncs of datasets
- $datadir = '/data/xmldatadumps'
- $publicdir = '/data/xmldatadumps/public'
- $tempdir = '/data/xmldatadumps/temp'
- $otherdir = '/data/otherdumps'
$cirrussearchdir = "${otherdir}/cirrussearch"
$xlationdir = "${otherdir}/contenttranslation"
$categoriesrdfdir = "${otherdir}/categoriesrdf"
@@ -24,7 +24,7 @@
$otherwikidatadir = "${otherdir}/wikidata"
# top level directories for various dumps/datasets
- file { [ $datadir, $publicdir, $otherdir, $tempdir ]:
+ file { [ $datadir, $xmldumpsdir, $otherdir, $tempdir ]:
ensure => 'directory',
mode => '0755',
owner => $user,
diff --git a/modules/dumps/manifests/rsync/public.pp
b/modules/dumps/manifests/rsync/public.pp
index 456a2d4..daa1723 100644
--- a/modules/dumps/manifests/rsync/public.pp
+++ b/modules/dumps/manifests/rsync/public.pp
@@ -1,6 +1,6 @@
class dumps::rsync::public(
$hosts_allow = undef,
- $publicdir = undef,
+ $xmldumpsdir = undef,
$otherdir = undef,
) {
file { '/etc/rsyncd.d/20-rsync-dumps_to_public.conf':
diff --git a/modules/dumps/manifests/web/cleanup.pp
b/modules/dumps/manifests/web/cleanup.pp
index 96fd3df..a1f85ba 100644
--- a/modules/dumps/manifests/web/cleanup.pp
+++ b/modules/dumps/manifests/web/cleanup.pp
@@ -1,7 +1,7 @@
class dumps::web::cleanup(
$miscdumpsdir = undef,
$isreplica = undef,
- $publicdir = undef,
+ $xmldumpsdir = undef,
$dumpstempdir = undef,
$user = undef,
) {
@@ -27,7 +27,7 @@
}
class {'::dumps::web::cleanups::xmldumps':
- publicdir => $publicdir,
+ xmldumpsdir => $xmldumpsdir,
dumpstempdir => $dumpstempdir,
user => $user,
isreplica => $isreplica,
diff --git a/modules/dumps/manifests/web/cleanups/xmldumps.pp
b/modules/dumps/manifests/web/cleanups/xmldumps.pp
index 534d681..fef1700 100644
--- a/modules/dumps/manifests/web/cleanups/xmldumps.pp
+++ b/modules/dumps/manifests/web/cleanups/xmldumps.pp
@@ -1,5 +1,5 @@
class dumps::web::cleanups::xmldumps(
- $publicdir = undef,
+ $xmldumpsdir = undef,
$dumpstempdir = undef,
$user = undef,
$isreplica = undef,
@@ -79,7 +79,7 @@
}
$xmlclean = '/usr/bin/python /usr/local/bin/cleanup_old_xmldumps.py'
- $args = "-d ${publicdir} -w ${wikilist_dir} -k /etc/dumps/xml_keeps.conf"
+ $args = "-d ${xmldumpsdir} -w ${wikilist_dir} -k /etc/dumps/xml_keeps.conf"
if ($isreplica == true) {
$cron_commands = "${xmlclean} ${args}"
diff --git a/modules/dumps/manifests/web/dirs.pp
b/modules/dumps/manifests/web/dirs.pp
index a85034c..b65cab7 100644
--- a/modules/dumps/manifests/web/dirs.pp
+++ b/modules/dumps/manifests/web/dirs.pp
@@ -1,7 +1,7 @@
class dumps::web::dirs(
- $datadir = '/data/xmldatadumps',
- $publicdir = '/data/xmldatadumps/public',
- $otherdir = '/data/xmldatadumps/public/other',
+ $datadir = undef,
+ $xmldumpsdir = undef,
+ $otherdir = undef,
$user = undef,
$group = undef,
) {
@@ -29,7 +29,7 @@
}
# top-level dirs for various dump trees
- file { [ $publicdir, $otherdir ]:
+ file { [ $xmldumpsdir, $otherdir ]:
ensure => 'directory',
mode => '0755',
owner => $user,
@@ -55,7 +55,7 @@
}
# subdirs for wikidata/wikibase weekly dumps
- file { [ $otherdir_wikibase, "${publicdir}/${relative_wikidatawiki}",
+ file { [ $otherdir_wikibase, "${xmldumpsdir}/${relative_wikidatawiki}",
$otherdir_wikidata_legacy ]:
ensure => 'directory',
@@ -64,7 +64,7 @@
group => $group,
}
# T72385: needs to be relative because it is mounted via NFS at differing
names
- file { "${publicdir}/wikidatawiki/entities":
+ file { "${xmldumpsdir}/wikidatawiki/entities":
ensure => 'link',
target => "../${relative_wikidatawiki}",
}
diff --git a/modules/dumps/manifests/web/fetches/kiwix.pp
b/modules/dumps/manifests/web/fetches/kiwix.pp
index 8c9397e..86fdb4c 100644
--- a/modules/dumps/manifests/web/fetches/kiwix.pp
+++ b/modules/dumps/manifests/web/fetches/kiwix.pp
@@ -1,12 +1,12 @@
class dumps::web::fetches::kiwix(
$user = undef,
$group = undef,
- $publicdir = undef,
+ $xmldumpsdir = undef,
$otherdir = undef,
) {
require_package('rsync')
- file { "${publicdir}/kiwix":
+ file { "${xmldumpsdir}/kiwix":
ensure => 'link',
target => "${otherdir}/kiwix",
owner => $user,
diff --git a/modules/dumps/manifests/web/html.pp
b/modules/dumps/manifests/web/html.pp
index 8f88964..f38b379 100644
--- a/modules/dumps/manifests/web/html.pp
+++ b/modules/dumps/manifests/web/html.pp
@@ -1,21 +1,21 @@
class dumps::web::html(
$datadir = undef,
- $publicdir = undef,
+ $xmldumpsdir = undef,
$otherdir = undef,
$webuser = undef,
$webgroup = undef,
) {
class {'dumps::web::dirs':
- datadir => $datadir,
- publicdir => $publicdir,
- otherdir => $otherdir,
- user => $webuser,
- group => $webgroup,
+ datadir => $datadir,
+ xmldumpsdir => $xmldumpsdir,
+ otherdir => $otherdir,
+ user => $webuser,
+ group => $webgroup,
}
- file { "${publicdir}/dumps.css":
+ file { "${xmldumpsdir}/dumps.css":
ensure => 'present',
- path => "${publicdir}/dumps.css",
+ path => "${xmldumpsdir}/dumps.css",
mode => '0644',
owner => 'root',
group => 'root',
@@ -40,9 +40,9 @@
source => 'puppet:///modules/dumps/web/html/pagecounts-ez_index.html',
}
- file { "${publicdir}/other/analytics/index.html":
+ file { "${xmldumpsdir}/other/analytics/index.html":
ensure => 'present',
- path => "${publicdir}/other/analytics/index.html",
+ path => "${xmldumpsdir}/other/analytics/index.html",
mode => '0644',
owner => 'root',
group => 'root',
@@ -58,36 +58,36 @@
source => 'puppet:///modules/dumps/web/html/poty_index.html',
}
- file { "${publicdir}/archive/index.html":
+ file { "${xmldumpsdir}/archive/index.html":
ensure => 'present',
- path => "${publicdir}/archive/index.html",
+ path => "${xmldumpsdir}/archive/index.html",
mode => '0644',
owner => 'root',
group => 'root',
source => 'puppet:///modules/dumps/web/html/archive_index.html',
}
- file { "${publicdir}/index.html":
+ file { "${xmldumpsdir}/index.html":
ensure => 'present',
- path => "${publicdir}/index.html",
+ path => "${xmldumpsdir}/index.html",
mode => '0644',
owner => 'root',
group => 'root',
source => 'puppet:///modules/dumps/web/html/public_index.html',
}
- file { "${publicdir}/mirrors.html":
+ file { "${xmldumpsdir}/mirrors.html":
ensure => 'present',
- path => "${publicdir}/mirrors.html",
+ path => "${xmldumpsdir}/mirrors.html",
mode => '0644',
owner => 'root',
group => 'root',
source => 'puppet:///modules/dumps/web/html/public_mirrors.html',
}
- file { "${publicdir}/legal.html":
+ file { "${xmldumpsdir}/legal.html":
ensure => 'present',
- path => "${publicdir}/legal.html",
+ path => "${xmldumpsdir}/legal.html",
mode => '0644',
owner => 'root',
group => 'root',
diff --git a/modules/dumps/manifests/web/xmldumps.pp
b/modules/dumps/manifests/web/xmldumps.pp
index 29ea54d..03587cb 100644
--- a/modules/dumps/manifests/web/xmldumps.pp
+++ b/modules/dumps/manifests/web/xmldumps.pp
@@ -2,7 +2,7 @@
class dumps::web::xmldumps(
$do_acme = true,
$datadir = undef,
- $publicdir = undef,
+ $xmldumpsdir = undef,
$otherdir = undef,
$htmldumps_server = undef,
$xmldumps_server = undef,
@@ -10,11 +10,11 @@
$webgroup = undef,
) {
class {'dumps::web::html':
- datadir => $datadir,
- publicdir => $publicdir,
- otherdir => $otherdir,
- webuser => $webuser,
- webgroup => $webgroup,
+ datadir => $datadir,
+ xmldumpsdir => $xmldumpsdir,
+ otherdir => $otherdir,
+ webuser => $webuser,
+ webgroup => $webgroup,
}
class { '::nginx':
@@ -35,7 +35,7 @@
}
include dumps::web::nginx_logrot
- file { "${publicdir}/favicon.ico":
+ file { "${xmldumpsdir}/favicon.ico":
source => 'puppet:///modules/dumps/web/xmldumps/favicon.ico',
owner => 'root',
group => 'root',
diff --git a/modules/dumps/manifests/web/xmldumps_active.pp
b/modules/dumps/manifests/web/xmldumps_active.pp
index e8ca5c8..99a96d7 100644
--- a/modules/dumps/manifests/web/xmldumps_active.pp
+++ b/modules/dumps/manifests/web/xmldumps_active.pp
@@ -1,7 +1,7 @@
class dumps::web::xmldumps_active(
$do_acme = true,
$datadir = undef,
- $publicdir = undef,
+ $xmldumpsdir = undef,
$otherdir = undef,
$logs_dest = undef,
$htmldumps_server = undef,
@@ -13,7 +13,7 @@
class {'::dumps::web::xmldumps':
do_acme => $do_acme,
datadir => $datadir,
- publicdir => $publicdir,
+ xmldumpsdir => $xmldumpsdir,
otherdir => $otherdir,
htmldumps_server => $htmldumps_server,
xmldumps_server => $xmldumps_server,
diff --git a/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
b/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
index f627d50..373a3ef 100644
--- a/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
+++ b/modules/dumps/templates/rsync/rsyncd.conf.dumps_to_public.erb
@@ -31,7 +31,7 @@
[dumpmirrorslastfivedumps]
# not really, we should transition people to using the module below
read only = true
-path = <%= @publicdir %>
+path = <%= @xmldumpsdir %>
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ *.inprog
hosts allow = <%= @hosts_allow %>
@@ -40,46 +40,46 @@
[dumpslastfive]
read only = true
# this includes only the last five good dumps.
-path = <%= @publicdir %>
+path = <%= @xmldumpsdir %>
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ *.inprog
/* /*/ /*/*/
-include from = <%= @publicdir %>/rsync-inc-last-5.txt
+include from = <%= @xmldumpsdir %>/rsync-inc-last-5.txt
hosts allow = <%= @hosts_allow %>
[dumpslastfour]
read only = true
# this includes only the last four good dumps.
-path = <%= @publicdir %>
+path = <%= @xmldumpsdir %>
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ *.inprog
/* /*/ /*/*/
-include from = <%= @publicdir %>/rsync-inc-last-4.txt
+include from = <%= @xmldumpsdir %>/rsync-inc-last-4.txt
hosts allow = <%= @hosts_allow %>
[dumpslastthree]
read only = true
# this includes only the last three good dumps.
-path = <%= @publicdir %>
+path = <%= @xmldumpsdir %>
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ *.inprog
/* /*/ /*/*/
-include from = <%= @publicdir %>/rsync-inc-last-3.txt
+include from = <%= @xmldumpsdir %>/rsync-inc-last-3.txt
hosts allow = <%= @hosts_allow %>
[dumpslasttwo]
read only = true
# this includes only the last two good dumps.
-path = <%= @publicdir %>
+path = <%= @xmldumpsdir %>
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ *.inprog
/* /*/ /*/*/
-include from = <%= @publicdir %>/rsync-inc-last-2.txt
+include from = <%= @xmldumpsdir %>/rsync-inc-last-2.txt
hosts allow = <%= @hosts_allow %>
[dumpslastone]
read only = true
# this includes only the last good dump.
-path = <%= @publicdir %>
+path = <%= @xmldumpsdir %>
include = /*wik*/
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ *.inprog
/* /*/ /*/*/
-include from = <%= @publicdir %>/rsync-inc-last-1.txt
+include from = <%= @xmldumpsdir %>/rsync-inc-last-1.txt
hosts allow = <%= @hosts_allow %>
[dumpmirrorsother]
@@ -91,14 +91,14 @@
[dumpmirrorsalldumps]
read only = true
# this includes only dumps, no archives, no other datasets
-path = <%= @publicdir %>
+path = <%= @xmldumpsdir %>
exclude = **tmp/ **temp/ **bad/ **save/ **other/ **archive/ **not/ *.inprog
hosts allow = <%= @hosts_allow %>
[dumpmirrorseverything]
read only = true
# this includes archives, other datasets
-path = <%= @publicdir %>
+path = <%= @xmldumpsdir %>
exclude = **tmp/ **temp/ **bad/ **save/ **not/ *.inprog
# ip addresses are: 69.31.98.2 your.org, 199.47.196.26 wansecurity.com,
69.28.137.74 iconicindustry.com
hosts allow = <%= @hosts_allow %> 199.47.196.26 69.31.98.2 69.28.137.74
diff --git a/modules/dumps/templates/web/xmldumps/nginx.conf.erb
b/modules/dumps/templates/web/xmldumps/nginx.conf.erb
index dafcbc5..031ee90 100644
--- a/modules/dumps/templates/web/xmldumps/nginx.conf.erb
+++ b/modules/dumps/templates/web/xmldumps/nginx.conf.erb
@@ -25,7 +25,7 @@
<%= @ssl_settings.join("\n ") %>
- root <%= @publicdir %>;
+ root <%= @xmldumpsdir %>;
location / {
index index.html index.htm;
diff --git a/modules/profile/manifests/dumps/fetcher.pp
b/modules/profile/manifests/dumps/fetcher.pp
index 24828f0..1210754 100644
--- a/modules/profile/manifests/dumps/fetcher.pp
+++ b/modules/profile/manifests/dumps/fetcher.pp
@@ -1,12 +1,12 @@
class profile::dumps::fetcher(
- $publicdir = hiera('profile::dumps::xmldumpspublicdir'),
+ $xmldumpsdir = hiera('profile::dumps::xmldumpspublicdir'),
$otherdir = hiera('profile::dumps::miscdumpsdir'),
) {
class {'dumps::web::fetches::kiwix':
- user => 'dumpsgen',
- group => 'dumpsgen',
- publicdir => $publicdir,
- otherdir => $otherdir,
+ user => 'dumpsgen',
+ group => 'dumpsgen',
+ xmldumpsdir => $xmldumpsdir,
+ otherdir => $otherdir,
}
class {'dumps::web::fetches::stats':
src => 'stat1005.eqiad.wmnet::hdfs-archive',
diff --git a/modules/profile/manifests/dumps/generation/server/fallback.pp
b/modules/profile/manifests/dumps/generation/server/fallback.pp
index cd8e303..644f4cc 100644
--- a/modules/profile/manifests/dumps/generation/server/fallback.pp
+++ b/modules/profile/manifests/dumps/generation/server/fallback.pp
@@ -2,7 +2,11 @@
class { '::dumpsuser': }
class { '::dumps::generation::server::dirs':
- user => $dumpsuser::user,
- group => $dumpsuser::group,
+ datadir => '/data/xmldatadumps',
+ xmldumpsdir => '/data/xmldatadumps/public',
+ tempdir => '/data/xmldatadumps/temp',
+ otherdir => '/data/otherdumps',
+ user => $dumpsuser::user,
+ group => $dumpsuser::group,
}
}
diff --git a/modules/profile/manifests/dumps/generation/server/primary.pp
b/modules/profile/manifests/dumps/generation/server/primary.pp
index 99af088..b6a4951 100644
--- a/modules/profile/manifests/dumps/generation/server/primary.pp
+++ b/modules/profile/manifests/dumps/generation/server/primary.pp
@@ -2,12 +2,16 @@
class { '::dumpsuser': }
class { '::dumps::generation::server::dirs':
- user => $dumpsuser::user,
- group => $dumpsuser::group,
+ datadir => '/data/xmldatadumps',
+ xmldumpsdir => '/data/xmldatadumps/public',
+ tempdir => '/data/xmldatadumps/temp',
+ otherdir => '/data/otherdumps',
+ user => $dumpsuser::user,
+ group => $dumpsuser::group,
}
class { '::dumps::generation::server::rsyncer':
- xmldumpsdir => '/data/xmldatadumps',
+ xmldumpsdir => '/data/xmldatadumps/public',
xmlremotedirs =>
'dumpsdata1002.eqiad.wmnet::data/xmldatadumps/public/,dataset1001.wikimedia.org::data/xmldatadumps/public/',
miscdumpsdir => '/data/otherdumps',
miscremotedirs =>
'dumpsdata1002.eqiad.wmnet::data/otherdumps/,dataset1001.wikimedia.org::data/xmldatadumps/public/other/',
diff --git a/modules/profile/manifests/dumps/rsyncer.pp
b/modules/profile/manifests/dumps/rsyncer.pp
index f264e92..61bef5e 100644
--- a/modules/profile/manifests/dumps/rsyncer.pp
+++ b/modules/profile/manifests/dumps/rsyncer.pp
@@ -14,9 +14,8 @@
$hosts_allow = join(concat($rsync_clients['ipv4']['external'],
$rsync_clients['ipv6']['external']), ' ')
- $xmldumpsdir = "${mntpoint}/xmldatadumps"
- $publicdir = "${xmldumpsdir}/public"
- $otherdir = "${publicdir}/other"
+ $xmldumpsdir = "${mntpoint}/xmldatadumps/public"
+ $otherdir = "${mntpoint}/xmldatadumps/public/other"
class {'::dumps::rsync::common':
user => $user,
@@ -53,7 +52,7 @@
class {'::dumps::rsync::public':
hosts_allow => $hosts_allow,
- publicdir => $publicdir,
+ xmldumpsdir => $xmldumpsdir,
otherdir => $otherdir,
}
diff --git a/modules/profile/manifests/dumps/web/cleanup.pp
b/modules/profile/manifests/dumps/web/cleanup.pp
index a4884cd..d1eedc5 100644
--- a/modules/profile/manifests/dumps/web/cleanup.pp
+++ b/modules/profile/manifests/dumps/web/cleanup.pp
@@ -1,13 +1,13 @@
class profile::dumps::web::cleanup(
$isreplica = hiera('profile::dumps::cleanup::isreplica'),
$miscdumpsdir = hiera('profile::dumps::miscdumpsdir'),
- $publicdir = hiera('profile::dumps::xmldumpspublicdir'),
+ $xmldumpsdir = hiera('profile::dumps::xmldumpspublicdir'),
$dumpstempdir = hiera('profile::dumps::dumpstempdir'),
) {
class {'::dumps::web::cleanup':
isreplica => $isreplica,
miscdumpsdir => $miscdumpsdir,
- publicdir => $publicdir,
+ xmldumpsdir => $xmldumpsdir,
dumpstempdir => $dumpstempdir,
user => 'dumpsgen',
}
diff --git a/modules/profile/manifests/dumps/web/dumpstatusfiles_sync.pp
b/modules/profile/manifests/dumps/web/dumpstatusfiles_sync.pp
index 72f476a..cbd77bc 100644
--- a/modules/profile/manifests/dumps/web/dumpstatusfiles_sync.pp
+++ b/modules/profile/manifests/dumps/web/dumpstatusfiles_sync.pp
@@ -3,6 +3,6 @@
) {
$mntpoint = $rsyncer_settings['dumps_mntpoint']
class {'::dumps::web::dumpstatusfiles':
- xmldumpsdir => "${mntpoint}/xmldatadumps",
+ xmldumpsdir => "${mntpoint}/xmldatadumps/public",
}
}
diff --git a/modules/profile/manifests/dumps/web/xmldumps_active.pp
b/modules/profile/manifests/dumps/web/xmldumps_active.pp
index e9d491c..82f5f36 100644
--- a/modules/profile/manifests/dumps/web/xmldumps_active.pp
+++ b/modules/profile/manifests/dumps/web/xmldumps_active.pp
@@ -2,13 +2,13 @@
$do_acme = hiera('do_acme'),
) {
class { '::dumpsuser': }
- $publicdir = '/data/xmldatadumps/public'
+ $xmldumpsdir = '/data/xmldatadumps/public'
$otherdir = '/data/xmldatadumps/public/other'
class {'::dumps::web::xmldumps_active':
do_acme => $do_acme,
datadir => '/data/xmldatadumps',
- publicdir => $publicdir,
+ xmldumpsdir => $xmldumpsdir,
otherdir => '/data/xmldatadumps/public/other',
logs_dest =>
'stat1005.eqiad.wmnet::srv/log/webrequest/archive/dumps.wikimedia.org/',
htmldumps_server => 'francium.eqiad.wmnet',
@@ -21,8 +21,8 @@
desthost => 'ms1001.wikimedia.org',
}
class {'::dumps::copying::labs':
- labhost => 'labstore1003.eqiad.wmnet',
- publicdir => $publicdir,
- otherdir => $otherdir,
+ labhost => 'labstore1003.eqiad.wmnet',
+ xmldumpsdir => $xmldumpsdir,
+ otherdir => $otherdir,
}
}
diff --git a/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
b/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
index 268410c..dedb83f 100644
--- a/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
+++ b/modules/profile/manifests/dumps/web/xmldumps_fallback.pp
@@ -6,7 +6,7 @@
class {'::dumps::web::xmldumps':
do_acme => $do_acme,
datadir => '/data/xmldatadumps',
- publicdir => '/data/xmldatadumps/public',
+ xmldumpsdir => '/data/xmldatadumps/public',
otherdir => '/data/xmldatadumps/public/other',
htmldumps_server => 'francium.eqiad.wmnet',
xmldumps_server => 'dumps.wikimedia.org',
--
To view, visit https://gerrit.wikimedia.org/r/397806
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ib3d75f53712c80bb2355a8aa8d0434502ae93eab
Gerrit-PatchSet: 6
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
Gerrit-Reviewer: Volans <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
