jenkins-bot has submitted this change and it was merged.
Change subject: [FEAT] upload: Support ignore warnings callback
......................................................................
[FEAT] upload: Support ignore warnings callback
This supports a callback method which is executed whenever an upload caused a
warning. This fixes the issue when multiple warnings are reported that the
upload method basically returned only one warning randomly.
It also provides a way to remove the success message as the method now returns
a boolean to indicate whether the upload succeeded.
Change-Id: I391bb2b7fd4a0f318c5b4537d4b615569413f447
---
M pywikibot/site.py
M scripts/upload.py
2 files changed, 141 insertions(+), 41 deletions(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/site.py b/pywikibot/site.py
index c09f399..3666f03 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -5180,7 +5180,7 @@
@deprecate_arg('imagepage', 'filepage')
def upload(self, filepage, source_filename=None, source_url=None,
comment=None, text=None, watch=False, ignore_warnings=False,
- chunk_size=0, _file_key=None, _offset=0):
+ chunk_size=0, _file_key=None, _offset=0, report_success=None):
"""Upload a file to the wiki.
Either source_filename or source_url, but not both, must be provided.
@@ -5195,8 +5195,16 @@
@param text: Initial page text; if this is not set, then
filepage.text will be used, or comment.
@param watch: If true, add filepage to the bot user's watchlist
- @param ignore_warnings: if true, ignore API warnings and force
- upload (for example, to overwrite an existing file); default False
+ @param ignore_warnings: It may be a static boolean, a callable
returning
+ a boolean or an iterable. The callable gets a list of UploadWarning
+ instances and the iterable should contain the warning codes for
+ which an equivalent callable would return True if all UploadWarning
+ codes are in thet list. If the result is False it'll not continuing
+ uploading the file and otherwise disable any warning and
+ reattempting to upload the file. NOTE: If report_success is True or
+ None it'll raise an UploadWarning exception if the static boolean
is
+ False.
+ @type ignore_warnings: bool or callable or iterable of str
@param chunk_size: The chunk size in bytesfor chunked uploading (see
U{https://www.mediawiki.org/wiki/API:Upload#Chunked_uploading}). It
will only upload in chunks, if the version number is 1.20 or higher
@@ -5209,7 +5217,23 @@
continue a previously canceled chunked upload. If False it treats
that as a finished upload. By default starts at 0.
@type _offset: int or bool
+ @param report_success: If the upload was successful it'll print a
+ success message and if ignore_warnings is set to False it'll
+ raise an UploadWarning if a warning occurred. If it's None
(default)
+ it'll be True if ignore_warnings is a bool and False otherwise. If
+ it's True or None ignore_warnings must be a bool.
+ @return: It returns True if the upload was successful and False
+ otherwise.
+ @rtype: bool
"""
+ def create_warnings_list(response):
+ return [
+ api.UploadWarning(
+ warning,
+ upload_warnings.get(warning, '%(msg)s') % {'msg': data},
+ _file_key, response.get('offset', 0))
+ for warning, data in response['warnings'].items()]
+
upload_warnings = {
# map API warning codes to user error messages
# %(msg)s will be replaced by message string from API responsse
@@ -5240,6 +5264,22 @@
if not comment:
raise ValueError("APISite.upload: cannot upload file without "
"a summary/description.")
+ if report_success is None:
+ report_success = isinstance(ignore_warnings, bool)
+ if report_success is True:
+ if not isinstance(ignore_warnings, bool):
+ raise ValueError('report_success may only be set to True when '
+ 'ignore_warnings is a boolean')
+ issue_deprecation_warning('"ignore_warnings" as a boolean and '
+ '"report_success" is True or None',
+ '"report_success=False" or define '
+ '"ignore_warnings" as callable/iterable',
+ 2)
+ if isinstance(ignore_warnings, Iterable):
+ ignored_warnings = ignore_warnings
+ ignore_warnings = lambda warnings: all(w.code in ignored_warnings
+ for w in warnings)
+ ignore_all_warnings = not callable(ignore_warnings) and ignore_warnings
if text is None:
text = filepage.text
if not text:
@@ -5289,7 +5329,7 @@
'filesize': filesize,
'offset': offset,
'filename': file_page_title,
- 'ignorewarnings': ignore_warnings})
+ 'ignorewarnings': ignore_all_warnings})
req.mime_params['chunk'] = (chunk,
("application",
"octet-stream"),
{'filename':
mime_filename})
@@ -5303,7 +5343,16 @@
if error.code == u'uploaddisabled':
self._uploaddisabled = True
raise error
- if 'warnings' in data and not ignore_warnings:
+ if 'warnings' in data and not ignore_all_warnings:
+ if callable(ignore_warnings):
+ if ignore_warnings(create_warnings_list(data)):
+ # Future warnings of this run can be
ignored
+ ignore_warnings = True
+ ignore_all_warnings = True
+ offset = result.get('offset', 0)
+ continue
+ else:
+ return False
result = data
if 'offset' not in result:
result['offset'] = 0
@@ -5345,7 +5394,7 @@
url=source_url, comment=comment, text=text, token=token)
if not result:
final_request['watch'] = watch
- final_request['ignorewarnings'] = ignore_warnings
+ final_request['ignorewarnings'] = ignore_all_warnings
try:
result = final_request.submit()
self._uploaddisabled = False
@@ -5357,10 +5406,7 @@
result = result["upload"]
pywikibot.debug(result, _logger)
- if "warnings" in result and not ignore_warnings:
- # TODO: Handle multiple warnings at the same time
- warning = list(result["warnings"].keys())[0]
- message = result["warnings"][warning]
+ if 'warnings' in result and not ignore_all_warnings:
if 'filekey' in result:
_file_key = result['filekey']
elif 'sessionkey' in result:
@@ -5370,6 +5416,24 @@
else:
_file_key = None
pywikibot.warning('No filekey defined.')
+ if not report_success:
+ if ignore_warnings(create_warnings_list(result)):
+ return self.upload(
+ filepage, source_filename, source_url, comment, text,
+ watch, True, chunk_size, _file_key,
+ result.get('offset', False), report_success=False)
+ else:
+ return False
+ warn('When ignore_warnings=False in APISite.upload will change '
+ 'from raising an UploadWarning into behaving like being a '
+ 'callable returning False.', DeprecationWarning, 2)
+ if len(result['warnings']) > 1:
+ warn('The upload returned {0} warnings: '
+ '{1}'.format(len(result['warnings']),
+ ', '.join(result['warnings'])),
+ UserWarning, 2)
+ warning = list(result["warnings"].keys())[0]
+ message = result["warnings"][warning]
raise pywikibot.UploadWarning(warning, upload_warnings[warning]
% {'msg': message},
file_key=_file_key,
@@ -5378,12 +5442,13 @@
elif "result" not in result:
pywikibot.output(u"Upload: unrecognized response: %s" % result)
if result["result"] == "Success":
- pywikibot.output(u"Upload successful.")
+ if report_success:
+ pywikibot.output(u"Upload successful.")
# If we receive a nochange, that would mean we're in simulation
# mode, don't attempt to access imageinfo
if "nochange" not in result:
filepage._load_file_revisions([result["imageinfo"]])
- return
+ return result['result'] == 'Success'
@deprecated_args(number='total',
repeat=None,
diff --git a/scripts/upload.py b/scripts/upload.py
index abf0ff7..5c50ab0 100755
--- a/scripts/upload.py
+++ b/scripts/upload.py
@@ -23,6 +23,13 @@
'Mi': Mebibytes (1024x1024 B)
The suffixes are case insensitive.
+It is possible to combine -abortonwarn and -ignorewarn so that if the specific
+warning is given it won't apply the general one but more specific one. So if it
+should ignore specific warnings and abort on the rest it's possible by defining
+no warning for -abortonwarn and the specific warnings for -ignorewarn. The
order
+does not matter. If both are unspecific or a warning is specified by both,
it'll
+prefer aborting.
+
If any other arguments are given, the first is either URL, filename or
directory
to upload, and the rest is a proposed description to go with the upload. If
none
of these are given, the user is asked for the directory, file or URL to upload.
@@ -197,6 +204,44 @@
t.close()
return tempname
+ def _handle_warning(self, warning):
+ """
+ Return whether the warning cause an abort or be ignored.
+
+ @param warning: The warning name
+ @type warning: str
+ @return: False if this warning should cause an abort, True if it should
+ be ignored or None if this warning has no default handler.
+ @rtype: bool or None
+ """
+ if self.aborts is not True:
+ if warning in self.aborts:
+ return False
+ if self.ignoreWarning is True or (self.ignoreWarning is not False and
+ warning in self.ignoreWarning):
+ return True
+ return None if self.aborts is not True else False
+
+ def _handle_warnings(self, warnings):
+ messages = '\n'.join('{0.code}: {0.info}'.format(warning)
+ for warning in sorted(warnings,
+ key=lambda w: w.code))
+ if len(warnings) > 1:
+ messages = '\n' + messages
+ pywikibot.output('We got the following warning(s): ' + messages)
+ answer = True
+ for warning in warnings:
+ this_answer = self._handle_warning(warning.code)
+ if this_answer is False:
+ answer = False
+ break
+ elif this_answer is None:
+ answer = None
+ if answer is None:
+ answer = pywikibot.input_yn(u"Do you want to ignore?",
+ default=False, automatic_quit=False)
+ return answer
+
def process_filename(self, file_url=None):
"""Return base filename portion of file_url."""
if not file_url:
@@ -351,54 +396,44 @@
pywikibot.output(u'Uploading file to %s via API...' % site)
+ success = False
try:
- apiIgnoreWarnings = False
if self.ignoreWarning is True:
apiIgnoreWarnings = True
+ else:
+ apiIgnoreWarnings = self._handle_warnings
if self.uploadByUrl:
- site.upload(imagepage, source_url=file_url,
- ignore_warnings=apiIgnoreWarnings,
- _file_key=_file_key, _offset=_offset)
+ success = site.upload(imagepage, source_url=file_url,
+ ignore_warnings=apiIgnoreWarnings,
+ _file_key=_file_key, _offset=_offset)
else:
if "://" in file_url:
temp = self.read_file_content(file_url)
else:
temp = file_url
- site.upload(imagepage, source_filename=temp,
- ignore_warnings=apiIgnoreWarnings,
- chunk_size=self.chunk_size,
- _file_key=_file_key, _offset=_offset)
+ success = site.upload(imagepage, source_filename=temp,
+ ignore_warnings=apiIgnoreWarnings,
+ chunk_size=self.chunk_size,
+ _file_key=_file_key, _offset=_offset)
- except pywikibot.data.api.UploadWarning as warn:
- pywikibot.output(
- u'We got a warning message: {0} - {1}'.format(warn.code,
warn.message))
- if self.abort_on_warn(warn.code):
- answer = False
- elif self.ignore_on_warn(warn.code):
- answer = True
- else:
- answer = pywikibot.input_yn(u"Do you want to ignore?",
- default=False,
automatic_quit=False)
- if answer:
- self.ignoreWarning = True
- self.keepFilename = True
- return self.upload_file(file_url, debug, warn.file_key,
warn.offset)
- else:
- pywikibot.output(u"Upload aborted.")
- return
except pywikibot.data.api.APIError as error:
if error.code == u'uploaddisabled':
pywikibot.error("Upload error: Local file uploads are disabled
on %s."
% site)
else:
pywikibot.error("Upload error: ", exc_info=True)
+ return None
except Exception:
pywikibot.error("Upload error: ", exc_info=True)
-
+ return None
else:
- # No warning, upload complete.
- pywikibot.output(u"Upload of %s successful." % filename)
- return filename # data['filename']
+ if success:
+ # No warning, upload complete.
+ pywikibot.output(u"Upload of %s successful." % filename)
+ return filename # data['filename']
+ else:
+ pywikibot.output(u"Upload aborted.")
+ return None
def run(self):
"""Run bot."""
--
To view, visit https://gerrit.wikimedia.org/r/233102
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I391bb2b7fd4a0f318c5b4537d4b615569413f447
Gerrit-PatchSet: 7
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: XZise <[email protected]>
Gerrit-Reviewer: John Vandenberg <[email protected]>
Gerrit-Reviewer: Ladsgroup <[email protected]>
Gerrit-Reviewer: XZise <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
Pywikibot-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/pywikibot-commits
