jenkins-bot has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/628339 )
Change subject: [IMPR] Improvements from download_dump.py
......................................................................
[IMPR] Improvements from download_dump.py
Change-Id: Ie011efd6c0243da00f0e76e3a5090d4cd527ccb3
---
M scripts/download_dump.py
1 file changed, 61 insertions(+), 64 deletions(-)
Approvals:
Hazard-SJ: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/download_dump.py b/scripts/download_dump.py
index 01f111d..a5f920a 100644
--- a/scripts/download_dump.py
+++ b/scripts/download_dump.py
@@ -32,7 +32,7 @@
"""Download dump bot."""
- availableOptions = { # noqa: N815
+ available_options = {
'wikiname': '',
'filename': '',
'storepath': './',
@@ -41,47 +41,43 @@
def get_dump_name(self, db_name, typ):
"""Check if dump file exists locally in a Toolforge server."""
- db_path = '/public/dumps/public/{0}/'.format(db_name)
+ db_path = '/public/dumps/public/{}/'.format(db_name)
if os.path.isdir(db_path):
dirs = [directory for directory in os.listdir(db_path) if
directory.isdigit()]
dates = map(int, dirs)
dates = sorted(dates, reverse=True)
for date in dates:
- dump_filepath = ('/public/dumps/public/{0}/{1}/{2}-{3}-{4}'
- .format(db_name, date, db_name, date, typ))
+ dump_filepath = '/public/dumps/public/{}/{}/{}-{}-{}'.format(
+ db_name, date, db_name, date, typ)
if os.path.isfile(dump_filepath):
return dump_filepath
return None
def run(self):
"""Run bot."""
- def convert_from_bytes(bytes):
+ def convert_from_bytes(total_bytes):
for unit in ['B', 'K', 'M', 'G', 'T']:
- if abs(bytes) < 1024:
- return str(bytes) + unit
- bytes = float(format(
- bytes / 1024.0, '.2f'))
- return str(bytes) + 'P'
+ if abs(total_bytes) < 1024:
+ return str(total_bytes) + unit
+ total_bytes = float(format(total_bytes / 1024.0, '.2f'))
+ return str(total_bytes) + 'P'
- pywikibot.output('Downloading dump from ' + self.getOption('wikiname'))
+ pywikibot.output('Downloading dump from ' + self.opt.wikiname)
- download_filename = '{wiki_name}-{dumpdate}-{filename}'.format(
- wiki_name=self.getOption('wikiname'),
- dumpdate=self.getOption('dumpdate'),
- filename=self.getOption('filename')
- )
+ download_filename = '{wikiname}-{dumpdate}-{filename}'.format_map(
+ self.opt)
temp_filename = download_filename + '-' + \
binascii.b2a_hex(urandom(8)).decode('ascii') + '.part'
file_final_storepath = os.path.join(
- self.getOption('storepath'), download_filename)
+ self.opt.storepath, download_filename)
file_current_storepath = os.path.join(
- self.getOption('storepath'), temp_filename)
+ self.opt.storepath, temp_filename)
# https://wikitech.wikimedia.org/wiki/Help:Toolforge#Dumps
toolforge_dump_filepath = self.get_dump_name(
- self.getOption('wikiname'), self.getOption('filename'))
+ self.opt.wikiname, self.opt.filename)
# First iteration for atomic download with temporary file
# Second iteration for fallback non-atomic download
@@ -95,9 +91,8 @@
remove(file_final_storepath)
symlink(toolforge_dump_filepath, file_current_storepath)
else:
- url = 'https://dumps.wikimedia.org/{0}/{1}/{2}'.format(
- self.getOption('wikiname'),
- self.getOption('dumpdate'),
+ url = 'https://dumps.wikimedia.org/{}/{}/{}'.format(
+ self.opt.wikiname, self.opt.dumpdate,
download_filename)
pywikibot.output('Downloading file from ' + url)
response = fetch(url, stream=True)
@@ -117,44 +112,45 @@
for data in response.data.iter_content(100 * 1024):
result_file.write(data)
- if total > 0:
- downloaded += len(data)
- done = int(parts * downloaded / total)
- display = map(convert_from_bytes,
- (downloaded, total))
- prior_display = display_string
- display_string = ('\r|{0}{1}|'
- '{2}{3}/{4}').format(
- '=' * done,
- '-' * (parts - done),
- ' ' * 5,
- * display)
- # Add whitespace to cover up prior bar
- display_string += ' ' * (
- len(prior_display.rstrip())
- - len(display_string.rstrip()))
+ if total <= 0:
+ continue
- pywikibot.output(display_string,
- newline=False)
+ downloaded += len(data)
+ done = int(parts * downloaded / total)
+ display = map(convert_from_bytes,
+ (downloaded, total))
+ prior_display = display_string
+ display_string = '\r|{}{}|{}{}/{}'.format(
+ '=' * done,
+ '-' * (parts - done),
+ ' ' * 5,
+ *display)
+ # Add whitespace to cover up prior bar
+ display_string += ' ' * (
+ len(prior_display.rstrip())
+ - len(display_string.rstrip()))
+
+ pywikibot.output(display_string, newline=False)
pywikibot.output('')
- elif response.status == 404:
- pywikibot.output(
- 'File with name "{filename}", '
- 'from dumpdate "{dumpdate}", '
- 'and wiki "{wikiname}" ({url}) isn\'t '
- 'available in the Wikimedia Dumps'.format(
- filename=self.getOption('filename'),
- dumpdate=self.getOption('dumpdate'),
- url=url,
- wikiname=self.getOption('wikiname')))
- return
else:
+ if response.status == 404:
+ pywikibot.output(
+ 'File with name "{filename}", '
+ 'from dumpdate "{dumpdate}", '
+ 'and wiki "{wikiname}" ({url}) isn\'t '
+ 'available in the Wikimedia Dumps'.format(
+ filename=self.opt.filename,
+ dumpdate=self.opt.dumpdate,
+ url=url,
+ wikiname=self.opt.wikiname))
return
+
# Rename the temporary file to the target file
# if the download completes successfully
if not non_atomic:
replace(file_current_storepath, file_final_storepath)
break
+
except (OSError, IOError):
pywikibot.exception()
@@ -165,15 +161,14 @@
# If the atomic download fails, try without a temporary file
# If the non-atomic download also fails, exit the script
- if not non_atomic:
- pywikibot.output('Cannot make temporary file, '
- 'falling back to non-atomic download')
- file_current_storepath = file_final_storepath
- else:
- return False
+ if non_atomic:
+ return
+
+ pywikibot.output('Cannot make temporary file, '
+ 'falling back to non-atomic download')
+ file_current_storepath = file_final_storepath
pywikibot.output('Done! File stored as ' + file_final_storepath)
- return
def main(*args):
@@ -193,24 +188,26 @@
option, _, value = arg.partition(':')
if option.startswith('-'):
option = option[1:]
+
if option == 'filename':
- opts[option] = value or pywikibot.input(
- 'Enter the filename: ')
+ opts[option] = value or pywikibot.input('Enter the filename: ')
continue
- elif option == 'storepath':
+
+ if option == 'storepath':
opts[option] = os.path.abspath(value) or pywikibot.input(
'Enter the store path: ')
continue
- elif option == 'dumpdate':
+
+ if option == 'dumpdate':
opts[option] = value or pywikibot.input(
'Enter the dumpdate of the dump: ')
continue
- unknown_args += [arg]
+ unknown_args.append(arg)
missing = []
if 'filename' not in opts:
- missing += ['-filename']
+ missing.append('-filename')
if pywikibot.bot.suggest_help(missing_parameters=missing,
unknown_parameters=unknown_args):
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/628339
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Ie011efd6c0243da00f0e76e3a5090d4cd527ccb3
Gerrit-Change-Number: 628339
Gerrit-PatchSet: 4
Gerrit-Owner: Xqt <[email protected]>
Gerrit-Reviewer: D3r1ck01 <[email protected]>
Gerrit-Reviewer: Hazard-SJ <[email protected]>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
_______________________________________________
Pywikibot-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/pywikibot-commits
