ArielGlenn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/385368 )
Change subject: clean up partially written files from previous 7z runs of same
wiki and date
......................................................................
clean up partially written files from previous 7z runs of same wiki and date
We write to files with .inprog extension now, these must be cleaned
up if the job is retried after failure, so that good content isn't
appended to these bad partial files.
Change-Id: I6e858fe11a34c115441a92cea18988514bc8942e
---
M xmldumps-backup/dumps/fileutils.py
M xmldumps-backup/dumps/jobs.py
M xmldumps-backup/dumps/recompressjobs.py
3 files changed, 59 insertions(+), 16 deletions(-)
Approvals:
ArielGlenn: Looks good to me, approved
jenkins-bot: Verified
diff --git a/xmldumps-backup/dumps/fileutils.py
b/xmldumps-backup/dumps/fileutils.py
index a53e76a..a264ece 100644
--- a/xmldumps-backup/dumps/fileutils.py
+++ b/xmldumps-backup/dumps/fileutils.py
@@ -150,6 +150,7 @@
is_checkpoint_file filename of form
dbname-date-dumpname-pxxxxpxxxx.xml.bz2
is_file_part filename of form dbname-date-dumpnamex.xml.gz/bz2/7z
is_temp_file filename of form dbname-date-dumpname.xml.gz/bz2/7z-tmp
+ is_inprog filename of form dbname-date-dumpname.<stuff>.inprog
first_page_id for checkpoint files, taken from value in filename
last_page_id for checkpoint files, value taken from filename
filename full filename
@@ -219,6 +220,7 @@
self.is_temp_file = False
self.temp = None
+ self.is_inprog = False
# example filenames:
# elwikidb-20110729-all-titles-in-ns0.gz
@@ -231,8 +233,15 @@
self.is_temp_file = True
self.temp = "-tmp"
- if '.' in self.filename:
- (file_base, self.file_ext) = self.filename.rsplit('.', 1)
+ if self.filename.endswith(".inprog"):
+ self.is_inprog = True
+
+ if self.is_inprog:
+ splitme = self.filename[:-7] # get rid of .inprog at end
+ else:
+ splitme = self.filename
+ if '.' in splitme:
+ (file_base, self.file_ext) = splitme.rsplit('.', 1)
if self.temp:
self.file_ext = self.file_ext[:-4]
else:
@@ -263,6 +272,8 @@
self.checkpoint_pattern = r"-p(?P<first>[0-9]+)p(?P<last>[0-9]+)\." +
self.file_ext
if self.temp is not None:
self.checkpoint_pattern += self.temp + "$"
+ elif self.is_inprog:
+ self.checkpoint_pattern += ".inprog$"
else:
self.checkpoint_pattern += "$"
@@ -725,7 +736,7 @@
def _get_files_filtered(self, date=None, dump_name=None, file_type=None,
file_ext=None, parts=None, temp=None,
checkpoint=None,
- skip_suffixes=None, required_suffixes=None):
+ skip_suffixes=None, required_suffixes=None,
inprog=False):
'''
list all files that exist, filtering by the given args.
if we get None for an arg then we accept all values
@@ -747,7 +758,6 @@
dfnames = self.get_files_in_dir(date)
dfnames_matched = []
for dfname in dfnames:
-
if skip_suffixes:
for suffix in skip_suffixes:
if dfname.filename.endswith(suffix):
@@ -776,6 +786,8 @@
continue
if (temp is False and dfname.is_temp_file) or (temp and not
dfname.is_temp_file):
continue
+ if (inprog is False and dfname.is_inprog) or (inprog is True and
not dfname.is_inprog):
+ continue
if ((checkpoint is False and dfname.is_checkpoint_file) or
(checkpoint and not dfname.is_checkpoint_file)):
continue
@@ -796,7 +808,7 @@
return mylist
def get_checkpt_files(self, date=None, dump_name=None,
- file_type=None, file_ext=None, parts=False,
temp=False):
+ file_type=None, file_ext=None, parts=False,
temp=False, inprog=False):
'''
list all checkpoint files that exist, filtering by the given args.
if we get None for an arg then we accept all values for that arg in
the filename
@@ -809,7 +821,7 @@
return self._get_files_filtered(date, dump_name, file_type,
file_ext, parts, temp, checkpoint=True,
required_suffixes=None,
- skip_suffixes=self.BAD)
+ skip_suffixes=self.BAD, inprog=inprog)
def get_truncated_empty_checkpt_files(self, date=None, dump_name=None,
file_type=None, file_ext=None,
@@ -829,7 +841,8 @@
skip_suffixes=None)
def get_reg_files(self, date=None, dump_name=None,
- file_type=None, file_ext=None, parts=False, temp=False,
suffix=None):
+ file_type=None, file_ext=None, parts=False, temp=False,
+ suffix=None, inprog=False):
'''
list all non-checkpoint files that exist, filtering by the given args.
if we get None for an arg then we accept all values for that arg in
the filename
@@ -851,4 +864,4 @@
return self._get_files_filtered(date, dump_name, file_type,
file_ext, parts, temp,
checkpoint=False,
required_suffixes=required_suffixes,
- skip_suffixes=skip_suffixes)
+ skip_suffixes=skip_suffixes,
inprog=inprog)
diff --git a/xmldumps-backup/dumps/jobs.py b/xmldumps-backup/dumps/jobs.py
index a3e4a29..8ebbbf1 100644
--- a/xmldumps-backup/dumps/jobs.py
+++ b/xmldumps-backup/dumps/jobs.py
@@ -96,9 +96,11 @@
command_info['output_dir'] = output_dir
else:
if runner.wiki.is_private():
- command_info['output_dir'] =
os.path.join(runner.wiki.private_dir(), runner.wiki.date)
+ command_info['output_dir'] =
os.path.join(runner.wiki.private_dir(),
+ runner.wiki.date)
else:
- command_info['output_dir'] =
os.path.join(runner.wiki.public_dir(), runner.wiki.date)
+ command_info['output_dir'] =
os.path.join(runner.wiki.public_dir(),
+ runner.wiki.date)
self.commands_submitted.append(command_info)
def check_truncation(self):
@@ -409,7 +411,7 @@
None, dname, self.file_type, self.file_ext, parts, temp=False))
return dfnames
- def list_checkpt_files_for_filepart(self, dump_dir, parts,
dump_names=None):
+ def list_checkpt_files_for_filepart(self, dump_dir, parts,
dump_names=None, inprog=False):
'''
list checkpoint files that have been produced for specified file
part(s)
returns:
@@ -420,10 +422,10 @@
dump_names = [self.dumpname]
for dname in dump_names:
dfnames.extend(dump_dir.get_checkpt_files(
- None, dname, self.file_type, self.file_ext, parts, temp=False))
+ None, dname, self.file_type, self.file_ext, parts, temp=False,
inprog=inprog))
return dfnames
- def list_reg_files_for_filepart(self, dump_dir, parts, dump_names=None):
+ def list_reg_files_for_filepart(self, dump_dir, parts, dump_names=None,
inprog=False):
'''
list noncheckpoint files that have been produced for specified file
part(s)
returns:
@@ -434,7 +436,7 @@
dump_names = [self.dumpname]
for dname in dump_names:
dfnames.extend(dump_dir.get_reg_files(
- None, dname, self.file_type, self.file_ext, parts, temp=False))
+ None, dname, self.file_type, self.file_ext, parts, temp=False,
inprog=inprog))
return dfnames
def list_truncated_empty_reg_files_for_filepart(self, dump_dir, parts,
dump_names=None):
diff --git a/xmldumps-backup/dumps/recompressjobs.py
b/xmldumps-backup/dumps/recompressjobs.py
index 68ed2b1..9d0a698 100644
--- a/xmldumps-backup/dumps/recompressjobs.py
+++ b/xmldumps-backup/dumps/recompressjobs.py
@@ -4,6 +4,7 @@
'''
from os.path import exists
+import os
from dumps.exceptions import BackupError
from dumps.fileutils import DumpFilename
from dumps.jobs import Dump
@@ -380,9 +381,36 @@
if errors:
raise BackupError("error recompressing bz2 file(s) %s")
+ def toss_inprog_files(self, dump_dir, runner):
+ """
+ delete partially written 7z files from previous failed attempts, if
+ any; 7z will otherwise blithely append onto them
+ """
+ if self.checkpoint_file is not None:
+ # we only rerun this one, so just remove this one
+ if exists(dump_dir.filename_public_path(self.checkpoint_file)):
+ if runner.dryrun:
+ print "would remove",
dump_dir.filename_public_path(self.checkpoint_file)
+ else:
+
os.remove(dump_dir.filename_public_path(self.checkpoint_file))
+ elif exists(dump_dir.filename_private_path(self.checkpoint_file)):
+ if runner.dryrun:
+ print "would remove",
dump_dir.filename_private_path(self.checkpoint_file)
+ else:
+
os.remove(dump_dir.filename_private_path(self.checkpoint_file))
+
+ dfnames = self.list_outfiles_for_cleanup(dump_dir)
+ if runner.dryrun:
+ print "would remove ", [dfname.filename for dfname in dfnames]
+ else:
+ for dfname in dfnames:
+ self.remove_output_file(dump_dir, dfname)
+
def run(self, runner):
commands = []
# Remove prior 7zip attempts; 7zip will try to append to an existing
archive
+ self.toss_inprog_files(runner.dump_dir, runner)
+
self.cleanup_old_files(runner.dump_dir, runner)
if self.checkpoint_file is not None:
output_dfname = DumpFilename(self.wiki, None,
self.checkpoint_file.dumpname,
@@ -489,12 +517,12 @@
dfnames = []
if self.item_for_recompression._checkpoints_enabled:
dfnames.extend(self.list_checkpt_files_for_filepart(
- dump_dir, self.get_fileparts_list(), dump_names))
+ dump_dir, self.get_fileparts_list(), dump_names, inprog=True))
dfnames.extend(self.list_temp_files_for_filepart(
dump_dir, self.get_fileparts_list(), dump_names))
else:
dfnames.extend(self.list_reg_files_for_filepart(
- dump_dir, self.get_fileparts_list(), dump_names))
+ dump_dir, self.get_fileparts_list(), dump_names, inprog=True))
return dfnames
def list_outfiles_for_input(self, dump_dir):
--
To view, visit https://gerrit.wikimedia.org/r/385368
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I6e858fe11a34c115441a92cea18988514bc8942e
Gerrit-PatchSet: 1
Gerrit-Project: operations/dumps
Gerrit-Branch: master
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
