jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/400616 )
Change subject: Made download_dump.py download process atomic
......................................................................
Made download_dump.py download process atomic
Bug: T183675
Change-Id: I142629bb89ffc1c810adcf8f1417ecc824594e41
---
M scripts/maintenance/download_dump.py
1 file changed, 66 insertions(+), 22 deletions(-)
Approvals:
Zhuyifei1999: Looks good to me, but someone else must approve
Framawiki: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/maintenance/download_dump.py
b/scripts/maintenance/download_dump.py
index d3a5123..c55940b 100644
--- a/scripts/maintenance/download_dump.py
+++ b/scripts/maintenance/download_dump.py
@@ -18,10 +18,27 @@
#
from __future__ import absolute_import, division, unicode_literals
+import binascii
+
import os.path
import sys
-from os import remove, symlink
+from os import remove, symlink, urandom
+
+try:
+ from os import replace
+except ImportError: # py2
+ if sys.platform == 'win32':
+ import os
+
+ def replace(src, dst):
+ try:
+ os.rename(src, dst)
+ except OSError:
+ remove(dst)
+ os.rename(src, dst)
+ else:
+ from os import rename as replace
import pywikibot
@@ -63,36 +80,63 @@
download_filename = self.getOption('wikiname') + \
'-latest-' + self.getOption('filename')
- file_storepath = os.path.join(
+ temp_filename = download_filename + '-' + \
+ binascii.b2a_hex(urandom(8)).decode('ascii') + '.part'
+
+ file_final_storepath = os.path.join(
self.getOption('storepath'), download_filename)
+ file_current_storepath = os.path.join(
+ self.getOption('storepath'), temp_filename)
# https://wikitech.wikimedia.org/wiki/Help:Toolforge#Dumps
toolforge_dump_filepath = self.get_dump_name(
self.getOption('wikiname'), self.getOption('filename'))
- if toolforge_dump_filepath:
- pywikibot.output('Symlinking file from ' + toolforge_dump_filepath)
- if os.path.exists(file_storepath):
- remove(file_storepath)
- symlink(toolforge_dump_filepath, file_storepath)
- else:
- url = 'https://dumps.wikimedia.org/' + \
- os.path.join(self.getOption('wikiname'),
- 'latest', download_filename)
- pywikibot.output('Downloading file from ' + url)
- response = fetch(url, stream=True)
- if response.status == 200:
+ # First iteration for atomic download with temporary file
+ # Second iteration for fallback non-atomic download
+ for non_atomic in range(2):
+ try:
+ if toolforge_dump_filepath:
+ pywikibot.output('Symlinking file from ' +
+ toolforge_dump_filepath)
+ if non_atomic:
+ if os.path.exists(file_final_storepath):
+ remove(file_final_storepath)
+ symlink(toolforge_dump_filepath, file_current_storepath)
+ else:
+ url = 'https://dumps.wikimedia.org/{0}/latest/{1}'.format(
+ self.getOption('wikiname'), download_filename)
+ pywikibot.output('Downloading file from ' + url)
+ response = fetch(url, stream=True)
+ if response.status == 200:
+ with open(file_current_storepath, 'wb') as result_file:
+ for data in response.data.iter_content(100 * 1024):
+ result_file.write(data)
+ else:
+ return
+ # Rename the temporary file to the target file
+ # if the download completes successfully
+ if not non_atomic:
+ replace(file_current_storepath, file_final_storepath)
+ break
+ except (OSError, IOError):
+ pywikibot.exception()
+
try:
- with open(file_storepath, 'wb') as result_file:
- for chunk in response.data.iter_content(100 * 1024):
- result_file.write(chunk)
- except IOError:
+ remove(file_current_storepath)
+ except (OSError, IOError):
pywikibot.exception()
- return False
- else:
- return
- pywikibot.output('Done! File stored as ' + file_storepath)
+ # If the atomic download fails, try without a temporary file
+ # If the non-atomic download also fails, exit the script
+ if not non_atomic:
+ pywikibot.output('Cannot make temporary file, ' +
+ 'falling back to non-atomic download')
+ file_current_storepath = file_final_storepath
+ else:
+ return False
+
+ pywikibot.output('Done! File stored as ' + file_final_storepath)
return
--
To view, visit https://gerrit.wikimedia.org/r/400616
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I142629bb89ffc1c810adcf8f1417ecc824594e41
Gerrit-PatchSet: 12
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Ryan10145 <chang.ryan10...@gmail.com>
Gerrit-Reviewer: Framawiki <framaw...@tools.wmflabs.org>
Gerrit-Reviewer: John Vandenberg <jay...@gmail.com>
Gerrit-Reviewer: Ryan10145 <chang.ryan10...@gmail.com>
Gerrit-Reviewer: Zhuyifei1999 <zhuyifei1...@gmail.com>
Gerrit-Reviewer: Zoranzoki21 <zorandori4...@gmail.com>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
