Zhuyifei1999 has uploaded a new change for review.
https://gerrit.wikimedia.org/r/277060
Change subject: [WIP] site: Support async uploads
......................................................................
[WIP] site: Support async uploads
This is still a work in progress.
Bug: T129754
Change-Id: Ifecb8ce8e6802a4e25fd1a5952cfa18ae37d037a
---
M pywikibot/site.py
1 file changed, 43 insertions(+), 22 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core
refs/changes/60/277060/1
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 4f37f8c..e182ca5 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -5596,8 +5596,8 @@
@deprecate_arg('imagepage', 'filepage')
def upload(self, filepage, source_filename=None, source_url=None,
comment=None, text=None, watch=False, ignore_warnings=False,
- chunk_size=0, _file_key=None, _offset=0, _verify_stash=None,
- report_success=None):
+ chunk_size=0, async=False, _file_key=None, _offset=0,
+ _verify_stash=None, report_success=None):
"""
Upload a file to the wiki.
@@ -5628,6 +5628,9 @@
will only upload in chunks, if the version number is 1.20 or higher
and the chunk size is positive but lower than the file size.
@type chunk_size: int
+ @param async: Make potentially large file operations asynchronous
+ on the server side when possible.
+ @type async: bool
@param _file_key: Reuses an already uploaded file using the filekey. If
None (default) it will upload the file.
@type _file_key: str or None
@@ -5714,6 +5717,7 @@
file_page_title = filepage.title(withNamespace=False)
file_size = None
offset = _offset
+ async = async and MediaWikiVersion(self.version()) >=
MediaWikiVersion('1.21')
# make sure file actually exists
if source_filename:
if os.path.isfile(source_filename):
@@ -5788,7 +5792,7 @@
final_request = self._simple_request(action='upload', token=token,
filename=file_page_title,
comment=comment, text=text,
- filekey=_file_key)
+ async=async,
filekey=_file_key)
elif source_filename:
# TODO: Dummy value to allow also Unicode names, see bug T75661
mime_filename = 'FAKE-NAME'
@@ -5801,7 +5805,8 @@
final_request = self._request(
throttle=throttle, parameters={
'action': 'upload', 'token': token, 'text': text,
- 'filename': file_page_title, 'comment': comment})
+ 'filename': file_page_title, 'comment': comment,
+ 'async': async})
if chunked_upload:
if offset > 0:
pywikibot.log('Continuing upload from byte '
@@ -5818,6 +5823,7 @@
'filesize': filesize,
'offset': offset,
'filename': file_page_title,
+ 'async': async,
'ignorewarnings': ignore_all_warnings})
req.mime_params['chunk'] = (chunk,
("application",
"octet-stream"),
@@ -5833,7 +5839,8 @@
self._uploaddisabled = True
raise error
_file_key = data['filekey']
- if 'warnings' in data and not ignore_all_warnings:
+ if data['result'] == 'Warning':
+ assert 'warnings' in data and not
ignore_all_warnings
if callable(ignore_warnings):
if 'offset' not in data:
data['offset'] = True
@@ -5849,23 +5856,37 @@
if 'offset' not in result:
result['offset'] = 0
break
- throttle = False
- if 'offset' in data:
- new_offset = int(data['offset'])
- if offset + len(chunk) != new_offset:
- pywikibot.log('Old offset: {0}; Returned '
- 'offset: {1}; Chunk size: '
- '{2}'.format(offset, new_offset,
- len(chunk)))
- pywikibot.warning('Unexpected offset.')
- offset = new_offset
- else:
- pywikibot.warning('Offset was not supplied.')
- offset += len(chunk)
- if data['result'] != 'Continue': # finished
+ elif data['result'] == 'Continue':
+ throttle = False
+ if 'offset' in data:
+ new_offset = int(data['offset'])
+ if offset + len(chunk) != new_offset:
+ pywikibot.log('Old offset: {0}; Returned '
+ 'offset: {1}; Chunk size: '
+ '{2}'.format(offset,
new_offset,
+ len(chunk)))
+ pywikibot.warning('Unexpected offset.')
+ offset = new_offset
+ else:
+ pywikibot.warning('Offset was not supplied.')
+ offset += len(chunk)
+ elif data['result'] == 'Success': # finished
pywikibot.log('Finished uploading last chunk.')
final_request['filekey'] = _file_key
break
+ elif data['result'] == 'Poll':
+ while True:
+ throttle = True
+ pywikibot.output('Waiting for server to '
+ 'assemble chunks.')
+ pollreq = self._simple_request(
+ action='upload', filekey=_file_key,
+ checkstatus=True)
+ polldata = pollreq.submit()
+ if data['result'] in ['Continue', 'Success']:
+ pass # TODO
+ else:
+ raise Error("Unrecognized result: %s" %
data['result'])
else: # not chunked upload
if _file_key:
final_request['filekey'] = _file_key
@@ -5884,8 +5905,8 @@
"User '%s' is not authorized to upload by URL on site %s."
% (self.user(), self))
final_request = self._simple_request(
- action='upload', filename=file_page_title,
- url=source_url, comment=comment, text=text, token=token)
+ action='upload', filename=file_page_title, url=source_url,
+ comment=comment, text=text, token=token, async=async)
if not result:
final_request['watch'] = watch
final_request['ignorewarnings'] = ignore_all_warnings
@@ -5916,7 +5937,7 @@
if ignore_warnings(create_warnings_list(result)):
return self.upload(
filepage, source_filename, source_url, comment, text,
- watch, True, chunk_size, _file_key,
+ watch, True, chunk_size, async, _file_key,
result['offset'], report_success=False)
else:
return False
--
To view, visit https://gerrit.wikimedia.org/r/277060
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ifecb8ce8e6802a4e25fd1a5952cfa18ae37d037a
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Zhuyifei1999 <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
