jenkins-bot has submitted this change and it was merged.
Change subject: [FEAT] upload: Support continue/finish uploads
......................................................................
[FEAT] upload: Support continue/finish uploads
When the upload was interrupted or didn't finished because of a warning it's
possible to reuse the already uploaded data and just publish it (if it was
finished) or upload the rest (when chunked mode is used).
Bug: T101657
Change-Id: I7f676b21856fbc8a65be7ced42a98dd1ca34d921
---
M pywikibot/data/api.py
M pywikibot/site.py
M scripts/upload.py
3 files changed, 63 insertions(+), 15 deletions(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py
index 236f58a..bec26a6 100644
--- a/pywikibot/data/api.py
+++ b/pywikibot/data/api.py
@@ -113,9 +113,20 @@
"""Upload failed with a warning message (passed as the argument)."""
- def __init__(self, code, message):
- """Constructor."""
+ def __init__(self, code, message, filekey=None, offset=0):
+ """
+ Create a new UploadWarning instance.
+
+ @param filekey: The filekey of the uploaded file to reuse it later. If
+ no key is known or it is an incomplete file it may be None.
+ @type filekey: str or None
+ @param offset: The starting offset for a chunked upload. Is False when
+ there is no offset.
+ @type offset: int or bool
+ """
super(UploadWarning, self).__init__(code, message)
+ self.filekey = filekey
+ self.offset = offset
@property
def message(self):
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 2c4f288..238c133 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -4959,7 +4959,7 @@
@deprecate_arg('imagepage', 'filepage')
def upload(self, filepage, source_filename=None, source_url=None,
comment=None, text=None, watch=False, ignore_warnings=False,
- chunk_size=0):
+ chunk_size=0, _file_key=None, _offset=0):
"""Upload a file to the wiki.
Either source_filename or source_url, but not both, must be provided.
@@ -4981,6 +4981,13 @@
will only upload in chunks, if the version number is 1.20 or higher
and the chunk size is positive but lower than the file size.
@type chunk_size: int
+ @param _file_key: Reuses an already uploaded file using the filekey. If
+ None (default) it will upload the file.
+ @type _file_key: str or None
+ @param _offset: When file_key is not None this can be an integer to
+ continue a previously canceled chunked upload. If False it treats
+ that as a finished upload. By default starts at 0.
+ @type _offset: int or bool
"""
upload_warnings = {
# map API warning codes to user error messages
@@ -4996,6 +5003,8 @@
'"%(msg)s".',
}
+ # An offset != 0 doesn't make sense without a file key
+ assert(_offset == 0 or _file_key is not None)
# check for required user right
if "upload" not in self.userinfo["rights"]:
raise Error(
@@ -5017,7 +5026,15 @@
token = self.tokens['edit']
result = None
file_page_title = filepage.title(withNamespace=False)
- if source_filename:
+ if _file_key and _offset is False:
+ pywikibot.log('Reused already upload file using '
+ 'filekey "{0}"'.format(_file_key))
+ # TODO: Use sessionkey instead of filekey if necessary
+ req = api.Request(site=self, action='upload', token=token,
+ filename=file_page_title,
+ comment=comment, text=text,
+ filekey=_file_key)
+ elif source_filename:
# TODO: Dummy value to allow also Unicode names, see bug 73661
mime_filename = 'FAKE-NAME'
# upload local file
@@ -5032,8 +5049,10 @@
MediaWikiVersion(self.version()) >=
MediaWikiVersion('1.20'))
with open(source_filename, 'rb') as f:
if chunked_upload:
- offset = 0
- file_key = None
+ offset = _offset
+ if offset > 0:
+ pywikibot.log('Continuing upload from byte '
+ '{0}'.format(offset))
while True:
f.seek(offset)
chunk = f.read(chunk_size)
@@ -5045,8 +5064,8 @@
req.mime_params['chunk'] = (chunk,
("application",
"octet-stream"),
{'filename':
mime_filename})
- if file_key:
- req['filekey'] = file_key
+ if _file_key:
+ req['filekey'] = _file_key
try:
data = req.submit()['upload']
self._uploaddisabled = False
@@ -5058,18 +5077,23 @@
if 'warnings' in data and not ignore_warnings:
result = data
break
- file_key = data['filekey']
+ _file_key = data['filekey']
throttle = False
if 'offset' in data:
new_offset = int(data['offset'])
if offset + len(chunk) != new_offset:
+ pywikibot.log('Old offset: {0}; Returned '
+ 'offset: {1}; Chunk size: '
+ '{2}'.format(offset, new_offset,
+ len(chunk)))
pywikibot.warning('Unexpected offset.')
offset = new_offset
else:
pywikibot.warning('Offset was not supplied.')
offset += len(chunk)
if data['result'] != 'Continue': # finished
- additional_parameters['filekey'] = file_key
+ pywikibot.log('Finished uploading last chunk.')
+ additional_parameters['filekey'] = _file_key
break
else: # not chunked upload
file_contents = f.read()
@@ -5113,8 +5137,19 @@
# TODO: Handle multiple warnings at the same time
warning = list(result["warnings"].keys())[0]
message = result["warnings"][warning]
+ if 'filekey' in result:
+ _file_key = result['filekey']
+ elif 'sessionkey' in result:
+ # TODO: Probably needs to be reflected in the API call above
+ _file_key = result['sessionkey']
+ pywikibot.warning('Using sessionkey instead of filekey.')
+ else:
+ _file_key = None
+ pywikibot.warning('No filekey defined.')
raise pywikibot.UploadWarning(warning, upload_warnings[warning]
- % {'msg': message})
+ % {'msg': message},
+ file_key=_file_key,
+ offset=result['offset'] if 'offset'
in result else 0)
elif "result" not in result:
pywikibot.output(u"Upload: unrecognized response: %s" % result)
if result["result"] == "Success":
diff --git a/scripts/upload.py b/scripts/upload.py
index 95c3a77..b7ea3fa 100755
--- a/scripts/upload.py
+++ b/scripts/upload.py
@@ -333,7 +333,7 @@
"""Upload image."""
self.upload_file(self.url, debug)
- def upload_file(self, file_url, debug=False):
+ def upload_file(self, file_url, debug=False, _file_key=None, _offset=0):
"""Upload the image at file_url to the target wiki.
Return the filename that was used to upload the image.
@@ -357,7 +357,8 @@
apiIgnoreWarnings = True
if self.uploadByUrl:
site.upload(imagepage, source_url=file_url,
- ignore_warnings=apiIgnoreWarnings)
+ ignore_warnings=apiIgnoreWarnings,
+ _file_key=_file_key, _offset=_offset)
else:
if "://" in file_url:
temp = self.read_file_content(file_url)
@@ -365,7 +366,8 @@
temp = file_url
site.upload(imagepage, source_filename=temp,
ignore_warnings=apiIgnoreWarnings,
- chunk_size=self.chunk_size)
+ chunk_size=self.chunk_size,
+ _file_key=_file_key, _offset=_offset)
except pywikibot.data.api.UploadWarning as warn:
pywikibot.output(
@@ -380,7 +382,7 @@
if answer:
self.ignoreWarning = True
self.keepFilename = True
- return self.upload_file(file_url, debug)
+ return self.upload_file(file_url, debug, warn.file_key,
warn.offset)
else:
pywikibot.output(u"Upload aborted.")
return
--
To view, visit https://gerrit.wikimedia.org/r/217275
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I7f676b21856fbc8a65be7ced42a98dd1ca34d921
Gerrit-PatchSet: 2
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: XZise <[email protected]>
Gerrit-Reviewer: John Vandenberg <[email protected]>
Gerrit-Reviewer: Ladsgroup <[email protected]>
Gerrit-Reviewer: Merlijn van Deen <[email protected]>
Gerrit-Reviewer: XZise <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
Pywikibot-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/pywikibot-commits
