ArielGlenn has uploaded a new change for review.
https://gerrit.wikimedia.org/r/278400
Change subject: dump url shorteners for wiki projects
......................................................................
dump url shorteners for wiki projects
All wikis have the same data so just choose one arbitrarily
This does a daily dump and adds a link to the directory on the
appropriate index.html page
Bug: T116986
Change-Id: I5c1c5878da1b72f9339950f3c78acc48670e3a40
---
M manifests/role/snapshot.pp
M modules/dataset/files/html/other_index.html
A modules/snapshot/manifests/dumps/shorturls.pp
3 files changed, 46 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/00/278400/1
diff --git a/manifests/role/snapshot.pp b/manifests/role/snapshot.pp
index 689a43e..22b3704 100644
--- a/manifests/role/snapshot.pp
+++ b/manifests/role/snapshot.pp
@@ -49,6 +49,10 @@
enable => true,
user => 'datasets',
}
+ class { 'snapshot::dumps::shorturls':
+ enable => true,
+ user => 'datasets',
+ }
class { 'snapshot::addschanges':
enable => true,
user => 'datasets',
diff --git a/modules/dataset/files/html/other_index.html
b/modules/dataset/files/html/other_index.html
index 627227f..af4efbb 100644
--- a/modules/dataset/files/html/other_index.html
+++ b/modules/dataset/files/html/other_index.html
@@ -121,6 +121,7 @@
<a href="mediacounts">Request counts for the upload
domain (pictures, movies, audio files)</a><br />
<a href="mediatitles">Titles of all files (namespace 6)
on each wiki, daily</a><br />
<a href="pagetitles">Titles of all articles (namespace
0) on each wiki, daily</a><br />
+ <a href="shorturls">Short urls used across all wiki
projects, daily</a><br />
<a href="kiwix/zim/">Static dumps of wiki projects in
OpenZim format (mirrored from Kiwix)</a><br />
<a href="surveys">WMF Survey data</a><br />
<a href="poty/">Picture of the Year Zip or GZ
files</a><br/>
diff --git a/modules/snapshot/manifests/dumps/shorturls.pp
b/modules/snapshot/manifests/dumps/shorturls.pp
new file mode 100644
index 0000000..da51574
--- /dev/null
+++ b/modules/snapshot/manifests/dumps/shorturls.pp
@@ -0,0 +1,41 @@
+class snapshot::dumps::shorturls(
+ $enable=true,
+ $user=undef,
+) {
+ include snapshot::dirs
+ include snapshot::wikiqueryskip
+
+ if ($enable) {
+ $ensure = 'present'
+ }
+ else {
+ $ensure = 'absent'
+ }
+
+ file { "${snapshot::dirs::datadir}/public/other/shorturls":
+ ensure => 'directory',
+ path => "${snapshot::dirs::datadir}/public/other/shorturls",
+ mode => '0755',
+ owner => $user,
+ group => root,
+ }
+
+ cron { 'shorturls-cleanup':
+ ensure => $ensure,
+ environment => '[email protected]',
+ user => $user,
+ command => "find
${snapshot::dirs::datadir}/public/other/shorturls/ -maxdepth 1 -type d -mtime
+90 -exec rm -rf {} \\;",
+ minute => '0',
+ hour => '8',
+ }
+
+ cron { 'shorturls':
+ ensure => $ensure,
+ environment => '[email protected]',
+ user => $user,
+ command => "cd ${snapshot::dirs::dumpsdir}; python onallwikis.py
--wiki bewikibooks --configfile confs/wikidump.conf.monitor --filenameformat
'shorturls-{d}.txt' --outdir
'${snapshot::dirs::datadir}/public/other/shorturls' --script
extensions/UrlShortener/maintenance/dumpURLs.php '{DIR}'",
+ minute => '5',
+ hour => '8',
+ require =>
File["${snapshot::dirs::datadir}/public/other/shorturls"],
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/278400
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I5c1c5878da1b72f9339950f3c78acc48670e3a40
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
