ArielGlenn has uploaded a new change for review.
https://gerrit.wikimedia.org/r/297399
Change subject: allow stubs to be done in serial batches of parallel jobs
......................................................................
allow stubs to be done in serial batches of parallel jobs
Bug: T132279
Change-Id: I9189d52b6d8d8657e2e7e20dfb38de55b24269b2
---
M xmldumps-backup/dumps/WikiDump.py
M xmldumps-backup/dumps/runner.py
M xmldumps-backup/dumps/xmljobs.py
3 files changed, 69 insertions(+), 11 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/dumps
refs/changes/99/297399/1
diff --git a/xmldumps-backup/dumps/WikiDump.py
b/xmldumps-backup/dumps/WikiDump.py
index 5e6a07f..f2b49f6 100644
--- a/xmldumps-backup/dumps/WikiDump.py
+++ b/xmldumps-backup/dumps/WikiDump.py
@@ -77,6 +77,9 @@
# "chunks": {
# set this to 1 to enable runing the various xml dump stages as
subjobs in parallel
"chunksEnabled": "0",
+ # do subjobs for named job in batches of specified size
+ # example: "xmlstubsdump=3,abstractsdump=5"
+ "jobsperbatch": "",
# for page history runs, number of pages for each file part,
specified separately
# e.g.
"1000,10000,100000,2000000,2000000,2000000,2000000,2000000,2000000,2000000"
# would define 10 subjobs with 10 file parts and the specified
number of pages in
@@ -257,6 +260,8 @@
self.conf.add_section('chunks')
self.parts_enabled = self.get_opt_for_proj_or_default(
conf, "chunks", "chunksEnabled", 1)
+ self.jobsperbatch = self.get_opt_for_proj_or_default(
+ conf, "chunks", "jobsperbatch", 0)
self.pages_per_filepart_history = self.get_opt_for_proj_or_default(
conf, "chunks", "pagesPerChunkHistory", 0)
self.revs_per_filepart_history = self.get_opt_for_proj_or_default(
diff --git a/xmldumps-backup/dumps/runner.py b/xmldumps-backup/dumps/runner.py
index 1752f64..b501c4a 100644
--- a/xmldumps-backup/dumps/runner.py
+++ b/xmldumps-backup/dumps/runner.py
@@ -61,6 +61,37 @@
self.queue.put_nowait(self.jobs_done)
+def get_setting(settings, setting_name):
+ '''
+ given a string of settings like "xmlstubsdump=this,xmldump=that",
+ return the value in the string for the specified setting name
+ or None if not present
+ '''
+ if '=' not in settings:
+ return None
+ if ',' in settings:
+ pairs = settings.split(',')
+ else:
+ pairs = [settings]
+ for pair in pairs:
+ if pair.startswith(setting_name + "="):
+ return pair.split('=')[1]
+ return None
+
+
+def get_int_setting(settings, setting_name):
+ '''
+ given a string of settings like "xmlstubsdump=num,xmldump=num",
+ return the int value in the string for the specified setting name
+ or None if not present
+ '''
+ value = get_setting(settings, setting_name)
+ if value is not None and value.isdigit():
+ return int(value)
+ else:
+ return None
+
+
class DumpItemList(object):
def __init__(self, wiki, prefetch, spawn, partnum_todo, checkpoint_file,
singleJob, skip_jobs, filepart, page_id_range, dumpjobdata,
dump_dir,
@@ -80,6 +111,7 @@
self.skip_jobs = skip_jobs
self.dumpjobdata = dumpjobdata
self.dump_dir = dump_dir
+ self.jobsperbatch = self.wiki.config.jobsperbatch
self.page_id_range = page_id_range
self.verbose = verbose
@@ -169,6 +201,7 @@
self.dump_items.append(XmlStub("xmlstubsdump", "First-pass for page
XML data dumps",
self._get_partnum_todo("xmlstubsdump"),
+ get_int_setting(self.jobsperbatch,
"xmlstubsdump"),
self.filepart.get_pages_per_filepart_history()))
self.append_job_if_needed(RecombineXmlStub(
diff --git a/xmldumps-backup/dumps/xmljobs.py b/xmldumps-backup/dumps/xmljobs.py
index c0bacf7..a6bade7 100644
--- a/xmldumps-backup/dumps/xmljobs.py
+++ b/xmldumps-backup/dumps/xmljobs.py
@@ -14,13 +14,26 @@
from dumps.jobs import Dump
+def batcher(items, batchsize):
+ '''
+ given a list of items and a batchsize, return
+ list of batches (lists) of these items, each
+ batch with batchsize items and the last batch
+ with however many items are left over
+ '''
+ if batchsize == 0:
+ return [items]
+ return (items[pos:pos + batchsize] for pos in xrange(0, len(items),
batchsize))
+
+
class XmlStub(Dump):
"""Create lightweight skeleton dumps, minus bulk text.
A second pass will import text from prior dumps or the database to make
full files for the public."""
- def __init__(self, name, desc, partnum_todo, parts=False,
checkpoints=False):
+ def __init__(self, name, desc, partnum_todo, jobsperbatch=None,
parts=False, checkpoints=False):
self._partnum_todo = partnum_todo
+ self.jobsperbatch = jobsperbatch
self._parts = parts
if self._parts:
self._parts_enabled = True
@@ -115,19 +128,25 @@
return series
def run(self, runner):
- commands = []
self.cleanup_old_files(runner.dump_dir, runner)
files = self.list_outfiles_for_build_command(runner.dump_dir)
- for fname in files:
- # choose arbitrarily one of the dump_names we do (=
articles_dump_name)
- # buildcommand will figure out the files for the rest
- if fname.dumpname == self.articles_dump_name:
+ # pick out the articles_dump files, setting up the stubs command for
these
+ # will cover all the other cases, as we generate all three stub file
types
+ # (article, meta-current, meta-history) at once
+ files = [fileinfo for fileinfo in files if fileinfo.dumpname ==
self.articles_dump_name]
+ if self.jobsperbatch is not None:
+ maxjobs = self.jobsperbatch
+ else:
+ maxjobs = len(files)
+ for batch in batcher(files, maxjobs):
+ commands = []
+ for fname in batch:
series = self.build_command(runner, fname)
commands.append(series)
- error = runner.run_command(commands,
callback_stderr=self.progress_callback,
- callback_stderr_arg=runner)
- if error:
- raise BackupError("error producing stub files")
+ error = runner.run_command(commands,
callback_stderr=self.progress_callback,
+ callback_stderr_arg=runner)
+ if error:
+ raise BackupError("error producing stub files")
class XmlLogging(Dump):
@@ -518,7 +537,8 @@
if not fileobj.filename or not
exists(runner.dump_dir.filename_public_path(fileobj)):
return None
- dumpfile = DumpFile(self.wiki,
runner.dump_dir.filename_public_path(fileobj, self.wiki.date),
+ dumpfile = DumpFile(self.wiki,
+ runner.dump_dir.filename_public_path(fileobj,
self.wiki.date),
fileobj, self.verbose)
pipeline = dumpfile.setup_uncompression_command()
--
To view, visit https://gerrit.wikimedia.org/r/297399
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I9189d52b6d8d8657e2e7e20dfb38de55b24269b2
Gerrit-PatchSet: 1
Gerrit-Project: operations/dumps
Gerrit-Branch: master
Gerrit-Owner: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
