XZise has uploaded a new change for review.
https://gerrit.wikimedia.org/r/217275
Change subject: [FEAT] upload: Support continue/finish uploads
......................................................................
[FEAT] upload: Support continue/finish uploads
When the upload was interrupted or didn't finished because of a warning it's
possible to reuse the already uploaded data and just publish it (if it was
finished) or upload the rest (when chunked mode is used).
Bug: T101657
Change-Id: I7f676b21856fbc8a65be7ced42a98dd1ca34d921
---
M pywikibot/data/api.py
M pywikibot/site.py
M scripts/upload.py
3 files changed, 56 insertions(+), 11 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core
refs/changes/75/217275/1
diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py
index 236f58a..bec26a6 100644
--- a/pywikibot/data/api.py
+++ b/pywikibot/data/api.py
@@ -113,9 +113,20 @@
"""Upload failed with a warning message (passed as the argument)."""
- def __init__(self, code, message):
- """Constructor."""
+ def __init__(self, code, message, filekey=None, offset=0):
+ """
+ Create a new UploadWarning instance.
+
+ @param filekey: The filekey of the uploaded file to reuse it later. If
+ no key is known or it is an incomplete file it may be None.
+ @type filekey: str or None
+ @param offset: The starting offset for a chunked upload. Is False when
+ there is no offset.
+ @type offset: int or bool
+ """
super(UploadWarning, self).__init__(code, message)
+ self.filekey = filekey
+ self.offset = offset
@property
def message(self):
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 2c4f288..ed49fff 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -4959,7 +4959,7 @@
@deprecate_arg('imagepage', 'filepage')
def upload(self, filepage, source_filename=None, source_url=None,
comment=None, text=None, watch=False, ignore_warnings=False,
- chunk_size=0):
+ chunk_size=0, file_key=None, offset=0):
"""Upload a file to the wiki.
Either source_filename or source_url, but not both, must be provided.
@@ -4981,6 +4981,13 @@
will only upload in chunks, if the version number is 1.20 or higher
and the chunk size is positive but lower than the file size.
@type chunk_size: int
+ @param file_key: Reuses an already uploaded file using the filekey. If
+ None (default) it will upload the file.
+ @type file_key: str or None
+ @param offset: When file_key is not None this can be an integer to
+ continue a previously canceled chunked upload. If False it treats
+ that as a finished upload. By default starts at 0.
+ @type offset: int or bool
"""
upload_warnings = {
# map API warning codes to user error messages
@@ -5017,7 +5024,14 @@
token = self.tokens['edit']
result = None
file_page_title = filepage.title(withNamespace=False)
- if source_filename:
+ if file_key and offset is False:
+ pywikibot.log('Reused already upload file using '
+ 'filekey "{0}"'.format(file_key))
+ req = api.Request(site=self, action='upload', token=token,
+ filename=file_page_title,
+ comment=comment, text=text,
+ filekey=file_key)
+ elif source_filename:
# TODO: Dummy value to allow also Unicode names, see bug 73661
mime_filename = 'FAKE-NAME'
# upload local file
@@ -5032,8 +5046,9 @@
MediaWikiVersion(self.version()) >=
MediaWikiVersion('1.20'))
with open(source_filename, 'rb') as f:
if chunked_upload:
- offset = 0
- file_key = None
+ if offset > 0:
+ pywikibot.log('Continuing upload from byte '
+ '{0}'.format(offset))
while True:
f.seek(offset)
chunk = f.read(chunk_size)
@@ -5063,13 +5078,19 @@
if 'offset' in data:
new_offset = int(data['offset'])
if offset + len(chunk) != new_offset:
+ pywikibot.log('Old offset: {0}; Returned '
+ 'offset: {1}; Chunk size: '
+ '{2}'.format(offset, new_offset,
+ len(chunk)))
pywikibot.warning('Unexpected offset.')
offset = new_offset
else:
pywikibot.warning('Offset was not supplied.')
offset += len(chunk)
if data['result'] != 'Continue': # finished
+ pywikibot.log('Finished uploading last chunk.')
additional_parameters['filekey'] = file_key
+ offset = False
break
else: # not chunked upload
file_contents = f.read()
@@ -5082,6 +5103,7 @@
{'filename': mime_filename})
}
}
+ offset = False
req = api.Request(site=self, action="upload", token=token,
filename=file_page_title,
comment=comment, text=text, throttle=throttle,
@@ -5095,6 +5117,7 @@
req = api.Request(site=self, action="upload", token=token,
filename=file_page_title,
url=source_url, comment=comment, text=text)
+ offset = False
if not result:
req['watch'] = watch
req['ignorewarnings'] = ignore_warnings
@@ -5113,8 +5136,10 @@
# TODO: Handle multiple warnings at the same time
warning = list(result["warnings"].keys())[0]
message = result["warnings"][warning]
+ # TODO: Determine when filekey was introduced
raise pywikibot.UploadWarning(warning, upload_warnings[warning]
- % {'msg': message})
+ % {'msg': message},
result['filekey'],
+ result['offset'] if 'offset' in
result else 0)
elif "result" not in result:
pywikibot.output(u"Upload: unrecognized response: %s" % result)
if result["result"] == "Success":
diff --git a/scripts/upload.py b/scripts/upload.py
index 95c3a77..7c9ab7b 100755
--- a/scripts/upload.py
+++ b/scripts/upload.py
@@ -333,7 +333,7 @@
"""Upload image."""
self.upload_file(self.url, debug)
- def upload_file(self, file_url, debug=False):
+ def upload_file(self, file_url, debug=False, upload_warning=None):
"""Upload the image at file_url to the target wiki.
Return the filename that was used to upload the image.
@@ -351,13 +351,21 @@
pywikibot.output(u'Uploading file to %s via API...' % site)
+ if upload_warning:
+ filekey = upload_warning.filekey
+ offset = upload_warning.offset
+ else:
+ filekey = None
+ offset = 0
+
try:
apiIgnoreWarnings = False
if self.ignoreWarning is True:
apiIgnoreWarnings = True
if self.uploadByUrl:
site.upload(imagepage, source_url=file_url,
- ignore_warnings=apiIgnoreWarnings)
+ ignore_warnings=apiIgnoreWarnings,
+ filekey=filekey, offset=offset)
else:
if "://" in file_url:
temp = self.read_file_content(file_url)
@@ -365,7 +373,8 @@
temp = file_url
site.upload(imagepage, source_filename=temp,
ignore_warnings=apiIgnoreWarnings,
- chunk_size=self.chunk_size)
+ chunk_size=self.chunk_size,
+ file_key=filekey, offset=offset)
except pywikibot.data.api.UploadWarning as warn:
pywikibot.output(
@@ -380,7 +389,7 @@
if answer:
self.ignoreWarning = True
self.keepFilename = True
- return self.upload_file(file_url, debug)
+ return self.upload_file(file_url, debug, warn)
else:
pywikibot.output(u"Upload aborted.")
return
--
To view, visit https://gerrit.wikimedia.org/r/217275
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7f676b21856fbc8a65be7ced42a98dd1ca34d921
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: XZise <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
