ArielGlenn has uploaded a new change for review.
https://gerrit.wikimedia.org/r/216404
Change subject: dumps: clean up error handling in xml streamed dumps
......................................................................
dumps: clean up error handling in xml streamed dumps
leave temp files around for investigation later
let exceptions do the work, and retry on exception
Change-Id: Ibb8a628afe00f69737e44a98f3b7196994e26614
---
M xmldumps-backup/xmlstreams.py
1 file changed, 26 insertions(+), 9 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/dumps
refs/changes/04/216404/1
diff --git a/xmldumps-backup/xmlstreams.py b/xmldumps-backup/xmlstreams.py
index 83f718e..c0d1bc3 100644
--- a/xmldumps-backup/xmlstreams.py
+++ b/xmldumps-backup/xmlstreams.py
@@ -105,6 +105,9 @@
if remainder != shouldendwith:
os.unlink(outfile)
failed = True
+ else:
+ failed = True
+
if failed:
return False
else:
@@ -116,9 +119,6 @@
read a file, cat it as fast as possible to the
stdin of the process passed, then go away
'''
- if not os.path.exists(inputfile):
- sys.stderr.write("no such file: %s\n" % inputfile)
- sys.exit(1)
with open(inputfile, "r") as filed:
while True:
content = filed.read(1048576)
@@ -194,7 +194,10 @@
maxretries = 3
timeout = 60
while retries < maxretries:
- result = run_script(command, outfiles, ends_with)
+ try:
+ result = run_script(command, outfiles, ends_with)
+ except:
+ result = False
if result:
break
time.sleep(timeout)
@@ -203,12 +206,26 @@
if not result:
sys.stderr.write("failed job after max retries\n")
for filetype in outfiles:
- os.unlink(outfiles[filetype]['temp'])
- # these partial output files can be used later with a
- # run that dumps the rest of the pages, and a recombine
- outfiles[filetype]['compr'].stdin.close()
+ try:
+ # these partial output files can be used later with a
+ # run that dumps the rest of the pages, and a recombine
+ # so we don't remove them
+ outfiles[filetype]['compr'].stdin.close()
+
+ # don't remove the temp files either, might be useful
+ # for checking the problem later
+ # os.unlink(outfiles[filetype]['temp'])
+ except:
+ pass
sys.exit(1)
for filetype in outfiles:
+ # any exception here means we don't unlink the temp files;
+ # this is intentional, might examine them or re-use them
catfile(outfiles[filetype]['temp'], outfiles[filetype]['compr'])
- os.unlink(outfiles[filetype]['temp'])
+
+ for filetype in outfiles:
+ try:
+ os.unlink(outfiles[filetype]['temp'])
+ except:
+ pass
--
To view, visit https://gerrit.wikimedia.org/r/216404
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibb8a628afe00f69737e44a98f3b7196994e26614
Gerrit-PatchSet: 1
Gerrit-Project: operations/dumps
Gerrit-Branch: ariel
Gerrit-Owner: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
