XZise has uploaded a new change for review.
https://gerrit.wikimedia.org/r/233102
Change subject: [FEAT] upload: Support ignore warnings callback
......................................................................
[FEAT] upload: Support ignore warnings callback
This supports a callback method which is executed whenever an upload caused a
warning.
Change-Id: I391bb2b7fd4a0f318c5b4537d4b615569413f447
---
M pywikibot/site.py
M scripts/upload.py
2 files changed, 135 insertions(+), 44 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core
refs/changes/02/233102/1
diff --git a/pywikibot/site.py b/pywikibot/site.py
index c09f399..76ad4b2 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -5196,7 +5196,10 @@
filepage.text will be used, or comment.
@param watch: If true, add filepage to the bot user's watchlist
@param ignore_warnings: if true, ignore API warnings and force
- upload (for example, to overwrite an existing file); default False
+ upload for example, to overwrite an existing file. It is by
default
+ False but can also be a callable getting a list of UploadWarning
+ instances which must either return True or False.
+ @type ignore_warnings: bool or callable
@param chunk_size: The chunk size in bytesfor chunked uploading (see
U{https://www.mediawiki.org/wiki/API:Upload#Chunked_uploading}). It
will only upload in chunks, if the version number is 1.20 or higher
@@ -5209,7 +5212,17 @@
continue a previously canceled chunked upload. If False it treats
that as a finished upload. By default starts at 0.
@type _offset: int or bool
+ @return: If C{ignore_warnings} is a bool it always returns None and
does
+ print a message that the upload was successful. If it's a callable
+ it returns True if the upload was successful and False otherwise.
+ @rtype: bool or None
"""
+ def warnings(response):
+ return [api.UploadWarning(
+ warning, upload_warnings[warning] % {'msg': data},
+ _file_key, response.get('offset', 0))
+ for warning, data in response['warnings'].items()]
+
upload_warnings = {
# map API warning codes to user error messages
# %(msg)s will be replaced by message string from API responsse
@@ -5289,7 +5302,8 @@
'filesize': filesize,
'offset': offset,
'filename': file_page_title,
- 'ignorewarnings': ignore_warnings})
+ 'ignorewarnings': not
callable(ignore_warnings) and
+ ignore_warnings})
req.mime_params['chunk'] = (chunk,
("application",
"octet-stream"),
{'filename':
mime_filename})
@@ -5303,11 +5317,20 @@
if error.code == u'uploaddisabled':
self._uploaddisabled = True
raise error
- if 'warnings' in data and not ignore_warnings:
- result = data
- if 'offset' not in result:
- result['offset'] = 0
- break
+ if 'warnings' in data:
+ if callable(ignore_warnings):
+ if ignore_warnings(warnings(data)):
+ # Future warnings of this run can be
ignored
+ ignore_warnings = True
+ offset = result.get('offset', 0)
+ continue
+ else:
+ return False
+ elif ignore_warnings:
+ result = data
+ if 'offset' not in result:
+ result['offset'] = 0
+ break
_file_key = data['filekey']
throttle = False
if 'offset' in data:
@@ -5345,7 +5368,8 @@
url=source_url, comment=comment, text=text, token=token)
if not result:
final_request['watch'] = watch
- final_request['ignorewarnings'] = ignore_warnings
+ final_request['ignorewarnings'] = (not callable(ignore_warnings)
and
+ ignore_warnings)
try:
result = final_request.submit()
self._uploaddisabled = False
@@ -5357,33 +5381,43 @@
result = result["upload"]
pywikibot.debug(result, _logger)
- if "warnings" in result and not ignore_warnings:
- # TODO: Handle multiple warnings at the same time
- warning = list(result["warnings"].keys())[0]
- message = result["warnings"][warning]
- if 'filekey' in result:
- _file_key = result['filekey']
- elif 'sessionkey' in result:
- # TODO: Probably needs to be reflected in the API call above
- _file_key = result['sessionkey']
- pywikibot.warning('Using sessionkey instead of filekey.')
- else:
- _file_key = None
- pywikibot.warning('No filekey defined.')
- raise pywikibot.UploadWarning(warning, upload_warnings[warning]
- % {'msg': message},
- file_key=_file_key,
- offset=result['offset']
- if 'offset' in result else
False)
- elif "result" not in result:
+ if "warnings" in result:
+ if callable(ignore_warnings):
+ if ignore_warnings(warnings(result)):
+ return self.upload(
+ filepage=filepage, source_filename=source_filename,
source_url=source_url,
+ comment=comment, text=text, watch=watch,
ignore_warnings=True,
+ chunk_size=chunk_size, _file_key=_file_key,
_offset=result.get('offset', False))
+ else:
+ return False
+ elif not ignore_warnings:
+ # TODO: Handle multiple warnings at the same time
+ warning = list(result["warnings"].keys())[0]
+ message = result["warnings"][warning]
+ if 'filekey' in result:
+ _file_key = result['filekey']
+ elif 'sessionkey' in result:
+ # TODO: Probably needs to be reflected in the API call
above
+ _file_key = result['sessionkey']
+ pywikibot.warning('Using sessionkey instead of filekey.')
+ else:
+ _file_key = None
+ pywikibot.warning('No filekey defined.')
+ raise pywikibot.UploadWarning(warning, upload_warnings[warning]
+ % {'msg': message},
+ file_key=_file_key,
+ offset=result['offset']
+ if 'offset' in result
else False)
+ if "result" not in result:
pywikibot.output(u"Upload: unrecognized response: %s" % result)
if result["result"] == "Success":
- pywikibot.output(u"Upload successful.")
+ if not callable(ignore_warnings):
+ pywikibot.output(u"Upload successful.")
# If we receive a nochange, that would mean we're in simulation
# mode, don't attempt to access imageinfo
if "nochange" not in result:
filepage._load_file_revisions([result["imageinfo"]])
- return
+ return result['result'] == 'Success' if callable(ignore_warnings) else
None
@deprecated_args(number='total',
repeat=None,
diff --git a/scripts/upload.py b/scripts/upload.py
index abf0ff7..c1607bb 100755
--- a/scripts/upload.py
+++ b/scripts/upload.py
@@ -23,6 +23,13 @@
'Mi': Mebibytes (1024x1024 B)
The suffixes are case insensitive.
+It is possible to combine -abortonwarn and -ignorewarn so that if the specific
+warning is given it won't apply the general one but more specific one. So if it
+should ignore specific warnings and abort on the rest it's possible by defining
+no warning for -abortonwarn and the specific warnings for -ignorewarn. The
order
+does not matter. If both are unspecific or a warning is specified by both,
it'll
+prefer aborting.
+
If any other arguments are given, the first is either URL, filename or
directory
to upload, and the rest is a proposed description to go with the upload. If
none
of these are given, the user is asked for the directory, file or URL to upload.
@@ -197,6 +204,44 @@
t.close()
return tempname
+ def _handle_warning(self, warning):
+ """
+ Return whether the warning cause an abort or be ignored.
+
+ @param warning: The warning name
+ @type warning: str
+ @return: False if this warning should cause an abort, True if it should
+ be ignored or None if this warning has no default handler.
+ @rtype: bool or None
+ """
+ if self.aborts is not True:
+ if warning in self.aborts:
+ return False
+ if self.ignoreWarning is True or warning in self.ignoreWarning:
+ return True
+ return None if self.aborts is not True else False
+
+ def _handle_warnings(self, warnings):
+ messages = '\n'.join('{0}: {1}'.format(warning.code, warning.info)
+ for warning in sorted(warnings))
+ if len(warnings) > 1:
+ messages = '\n' + messages
+ pywikibot.output('We got the following warning(s): ' + messages)
+ answer = True
+ for warning in warnings:
+ this_answer = self._handle_warning(warning)
+ if this_answer is False:
+ answer = False
+ break
+ elif this_answer is None:
+ answer = None
+ if answer is None:
+ answer = pywikibot.input_yn(u"Do you want to ignore?",
+ default=False, automatic_quit=False)
+ if not answer:
+ pywikibot.output(u"Upload aborted.")
+ return answer
+
def process_filename(self, file_url=None):
"""Return base filename portion of file_url."""
if not file_url:
@@ -351,32 +396,42 @@
pywikibot.output(u'Uploading file to %s via API...' % site)
+ success = False
try:
- apiIgnoreWarnings = False
if self.ignoreWarning is True:
apiIgnoreWarnings = True
+ else:
+ apiIgnoreWarnings = self._handle_warnings
if self.uploadByUrl:
- site.upload(imagepage, source_url=file_url,
- ignore_warnings=apiIgnoreWarnings,
- _file_key=_file_key, _offset=_offset)
+ success = site.upload(imagepage, source_url=file_url,
+ ignore_warnings=apiIgnoreWarnings,
+ _file_key=_file_key, _offset=_offset)
else:
if "://" in file_url:
temp = self.read_file_content(file_url)
else:
temp = file_url
- site.upload(imagepage, source_filename=temp,
- ignore_warnings=apiIgnoreWarnings,
- chunk_size=self.chunk_size,
- _file_key=_file_key, _offset=_offset)
+ success = site.upload(imagepage, source_filename=temp,
+ ignore_warnings=apiIgnoreWarnings,
+ chunk_size=self.chunk_size,
+ _file_key=_file_key, _offset=_offset)
except pywikibot.data.api.UploadWarning as warn:
+ messages = '\n'.join('{0}: {1}'.format(code, msg)
+ for code, msg in
sorted(warn.messages.items()))
+ if len(warn.messages) > 1:
+ messages = '\n' + messages
pywikibot.output(
- u'We got a warning message: {0} - {1}'.format(warn.code,
warn.message))
- if self.abort_on_warn(warn.code):
- answer = False
- elif self.ignore_on_warn(warn.code):
- answer = True
- else:
+ 'We got the following warning(s): ' + messages)
+ answer = True
+ for code in warn.messages:
+ this_answer = self._handle_warning(code)
+ if this_answer is False:
+ answer = False
+ break
+ elif this_answer is None:
+ answer = None
+ if answer is None:
answer = pywikibot.input_yn(u"Do you want to ignore?",
default=False,
automatic_quit=False)
if answer:
@@ -395,10 +450,12 @@
except Exception:
pywikibot.error("Upload error: ", exc_info=True)
- else:
+ if success:
# No warning, upload complete.
pywikibot.output(u"Upload of %s successful." % filename)
return filename # data['filename']
+ else:
+ return None
def run(self):
"""Run bot."""
--
To view, visit https://gerrit.wikimedia.org/r/233102
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I391bb2b7fd4a0f318c5b4537d4b615569413f447
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: XZise <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
