ArielGlenn has uploaded a new change for review.
https://gerrit.wikimedia.org/r/66666
Change subject: dump all page titles in all namespaces (bug #19542)
......................................................................
dump all page titles in all namespaces (bug #19542)
Change-Id: I7f53f1eb2f4396d6fc9f80625919a6c745bfa21f
---
M xmldumps-backup/worker.py
1 file changed, 27 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/dumps
refs/changes/66/66666/1
diff --git a/xmldumps-backup/worker.py b/xmldumps-backup/worker.py
index 3cfe9a9..861ba08 100644
--- a/xmldumps-backup/worker.py
+++ b/xmldumps-backup/worker.py
@@ -580,6 +580,7 @@
self._singleJob == 'latestlinks' or
self._singleJob == 'xmlpagelogsdump' or
self._singleJob == 'pagetitlesdump' or
+ self._singleJob == 'allpagetitlesdump' or
self._singleJob.endswith('recombine')):
raise BackupError("You cannot specify a chunk
with the job %s, exiting.\n" % self._singleJob)
@@ -590,6 +591,7 @@
self._singleJob == 'latestlinks' or
self._singleJob == 'xmlpagelogsdump' or
self._singleJob == 'pagetitlesdump' or
+ self._singleJob == 'allpagetitlesdump' or
self._singleJob == 'abstractsdump' or
self._singleJob == 'xmlstubsdump' or
self._singleJob.endswith('recombine')):
@@ -626,7 +628,8 @@
PublicTable("redirect", "redirecttable", "Redirect
list"),
PublicTable("iwlinks", "iwlinkstable", "Interwiki link
tracking records"),
- TitleDump("pagetitlesdump", "List of page titles"),
+ TitleDump("pagetitlesdump", "List of page titles in
main namespace"),
+ AllTitleDump("allpagetitlesdump", "List of all page
titles"),
AbstractDump("abstractsdump","Extracted page abstracts
for Yahoo", self._getChunkToDo("abstractsdump"),
self.chunkInfo.getPagesPerChunkAbstract())]
@@ -2691,7 +2694,7 @@
def run(self, runner):
retries = 0
# try this initially and see how it goes
- maxretries = 3
+ maxretries = 3
files = self.listOutputFilesForBuildCommand(runner.dumpDir)
if (len(files) > 1):
raise BackupError("table dump %s trying to produce more
than one file" % self.dumpName)
@@ -4025,6 +4028,28 @@
command = runner.dbServerInfo.buildSqlCommand(query,
runner.wiki.config.gzip)
return runner.saveCommand(command, outfile)
+class AllTitleDump(TitleDump):
+
+ def getDumpName(self):
+ return "all-titles"
+
+ def run(self, runner):
+ retries = 0
+ maxretries = 3
+ query="select page_title from page;"
+ files = self.listOutputFilesForBuildCommand(runner.dumpDir)
+ if len(files) > 1:
+ raise BackupError("all titles dump trying to produce
more than one output file")
+ fileObj = files[0]
+ outFilename = runner.dumpDir.filenamePublicPath(fileObj)
+ error = self.saveSql(query, outFilename, runner)
+ while (error and retries < maxretries):
+ retries = retries + 1
+ time.sleep(5)
+ error = self.saveSql(query, outFilename, runner)
+ if (error):
+ raise BackupError("error dumping all titles list")
+
def findAndLockNextWiki(config, locksEnabled, cutoff):
if config.halt:
sys.stderr.write("Dump process halted by config.\n")
--
To view, visit https://gerrit.wikimedia.org/r/66666
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7f53f1eb2f4396d6fc9f80625919a6c745bfa21f
Gerrit-PatchSet: 1
Gerrit-Project: operations/dumps
Gerrit-Branch: ariel
Gerrit-Owner: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
