Xqt has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1325981?usp=email )
Change subject: upload: Replace recursive orchestration
......................................................................
upload: Replace recursive orchestration
Model upload progress with explicit private states so warning
retries and stash resumptions no longer re-enter _upload().
Separate preparation, chunk transfer, polling, and publication
while preserving existing upload behavior.
Bug: T434936
Change-Id: I95eb1f88d6a957e619c286b2a67626cf43ac343a
---
M pywikibot/site/_upload.py
1 file changed, 492 insertions(+), 346 deletions(-)
Approvals:
Xqt: Verified; Looks good to me, approved
diff --git a/pywikibot/site/_upload.py b/pywikibot/site/_upload.py
index 0e605d3..543b66e 100644
--- a/pywikibot/site/_upload.py
+++ b/pywikibot/site/_upload.py
@@ -9,6 +9,9 @@
import mimetypes
import os
from collections.abc import Iterable
+from dataclasses import dataclass
+from enum import Enum, auto
+from typing import Any
from warnings import warn
import pywikibot
@@ -19,6 +22,49 @@
__all__ = ('Uploader', )
+class _UploadStage(Enum):
+
+ """Stages of an upload attempt."""
+
+ PREPARE = auto()
+ RECOVER_STASH = auto()
+ TRANSFER = auto()
+ POLL_TRANSFER = auto()
+ FINALIZE = auto()
+ POLL_PUBLICATION = auto()
+
+
+class _ResumeMode(Enum):
+
+ """Ways to resume an upload from the server stash."""
+
+ FRESH = auto()
+ DISCOVER_OFFSET = auto()
+ AT_OFFSET = auto()
+ COMPLETE_STASH = auto()
+
+
+@dataclass
+class _UploadState:
+
+ """Mutable progress for one upload invocation."""
+
+ ignore_warnings: Any
+ ignore_all_warnings: bool
+ report_success: bool | None
+ stage: _UploadStage = _UploadStage.PREPARE
+ resume_mode: _ResumeMode = _ResumeMode.FRESH
+ file_key: str | None = None
+ offset: int = 0
+ file_size: int | None = None
+ token: str | None = None
+ file_page_title: str | None = None
+ request: Any = None
+ response: dict[str, Any] | None = None
+ data_result: str | None = None
+ throttle: bool = True
+
+
class Uploader:
"""Uploader class to upload a file to the wiki.
@@ -150,20 +196,13 @@
def _upload(self, ignore_warnings, report_success,
file_key=None, offset=0) -> bool:
- """Recursive Upload method.
+ """Upload the file using explicit state transitions."""
+ state = self._new_state(ignore_warnings, report_success)
+ self._restart_upload(state, file_key, offset)
+ return self._run_upload(state)
- :param file_key: Reuses an already uploaded file using the
- filekey. If None (default) it will upload the file.
- :param offset: When file_key is not None this can be an integer
- to continue a previously canceled chunked upload. If False
- it treats that as a finished upload. If True it requests the
- stash info from the server to determine the offset. By
- default starts at 0.
- :return: Whether the upload was successful.
- """
- # An offset != 0 doesn't make sense without a file key
- assert offset == 0 or file_key is not None
-
+ def _new_state(self, ignore_warnings, report_success) -> _UploadState:
+ """Normalize upload options and create its initial state."""
if report_success is None:
report_success = isinstance(ignore_warnings, bool)
if report_success is True and not isinstance(ignore_warnings, bool):
@@ -175,358 +214,465 @@
def ignore_warnings(warnings):
return all(w.code in ignored_warnings for w in warnings)
- ignore_all_warnings = not callable(ignore_warnings) and ignore_warnings
+ ignore_all_warnings = (
+ not callable(ignore_warnings) and ignore_warnings)
+ return _UploadState(ignore_warnings, ignore_all_warnings,
+ report_success)
- token = self.site.tokens['csrf']
- result = None
- file_page_title = self.filepage.title(
+ def _run_upload(self, state: _UploadState) -> bool:
+ """Run upload stages until the upload completes or aborts."""
+ while True:
+ if state.stage is _UploadStage.PREPARE:
+ self._prepare_upload(state)
+ elif state.stage is _UploadStage.RECOVER_STASH:
+ self._recover_stash(state)
+ elif state.stage in {
+ _UploadStage.TRANSFER,
+ _UploadStage.POLL_TRANSFER,
+ }:
+ outcome = self._transfer_chunks(state)
+ if outcome is not None:
+ return outcome
+ elif state.stage in {
+ _UploadStage.FINALIZE,
+ _UploadStage.POLL_PUBLICATION,
+ }:
+ outcome = self._finalize_upload(state)
+ if outcome is not None:
+ return outcome
+ else: # pragma: no cover - all stages are handled above
+ raise AssertionError(f'Unknown upload stage: {state.stage}')
+
+ def _prepare_upload(self, state: _UploadState) -> None:
+ """Prepare one fresh or resumed upload attempt."""
+ state.token = self.site.tokens['csrf']
+ state.file_page_title = self.filepage.title(
with_ns=False,
with_section=False,
)
- file_size = None
+ state.file_size = None
+ state.request = None
+ state.response = None
+ state.data_result = None
+ state.throttle = True
- # make sure file actually exists
if self.filename:
if os.path.isfile(self.filename):
- file_size = os.path.getsize(self.filename)
- elif offset is not False:
+ state.file_size = os.path.getsize(self.filename)
+ elif state.resume_mode is not _ResumeMode.COMPLETE_STASH:
raise ValueError(f"File '{self.filename}' does not exist.")
- # Verify the stash when a file key and offset is given:
- # requests the SHA1 and file size uploaded and compares it to
- # the local file. Also verify that offset is matching the
- # file size if the offset is an int. If offset is False if
- # verifies that the file size match with the local file.
- verify_stash = False
- if self.filename and file_key:
- assert offset is False or file_size is not None
- verify_stash = True
- if (offset is not False and offset is not True
- and offset > file_size):
- raise ValueError(
- f'For the file key "{file_key}" the offset was set to '
- f'{offset} while the file is only {file_size} bytes large.'
- )
-
- if verify_stash or offset is True:
- if not file_key:
- raise ValueError('Without a file key it cannot request the '
- 'stash information')
- if not self.filename:
- raise ValueError('Can request stash information only when '
- 'using a file name.')
- props = ['size']
- if verify_stash:
- props.append('sha1')
- stash_info = self.site.stash_info(file_key, props)
- if offset is True:
- offset = stash_info['size']
- elif offset is False:
- if file_size != stash_info['size']:
- raise ValueError(
- f'For the file key "{file_key}" the server reported '
- f'a size {stash_info["size"]} while the file size is '
- f'{file_size}')
- elif offset is not False and offset != stash_info['size']:
- raise ValueError(
- f'For the file key "{file_key}" the server reported '
- f'a size {stash_info["size"]} while the offset was '
- f'{offset}')
-
- if verify_stash:
- # The SHA1 was also requested so calculate and compare it
- assert 'sha1' in stash_info, \
- f'sha1 not in stash info: {stash_info}'
- sha1 = compute_file_hash(
- self.filename,
- bytes_to_read=None if offset is False else offset)
- if sha1 != stash_info['sha1']:
- raise ValueError(
- f'The SHA1 of {offset} bytes of the stashed '
- f'"{file_key}" is {stash_info["sha1"]} '
- f'while the local file is {sha1}')
-
- assert offset is not True
- if file_key and file_size is None:
- assert offset is False
-
- data = {}
- if file_key and offset is False or offset == file_size:
- pywikibot.log(
- f'Reused already upload file using filekey "{file_key}"')
- # TODO: Use sessionkey instead of filekey if necessary
- final_request = self.site._request(
- parameters={
- 'action': 'upload',
- 'token': token,
- 'filename': file_page_title,
- 'comment': self.comment,
- 'text': self.text,
- 'async': self.asynchronous,
- 'filekey': file_key
- })
-
- elif self.filename:
- # TODO: Dummy value to allow also Unicode names, see bug T75661
- mime_filename = 'FAKE-NAME'
- # upload local file
- throttle = True
- filesize = os.path.getsize(self.filename)
- chunked_upload = 0 < self.chunk_size < filesize
- with open(self.filename, 'rb') as f:
- final_request = self.site._request(
- throttle=throttle, parameters={
- 'action': 'upload', 'token': token, 'text': self.text,
- 'filename': file_page_title, 'comment': self.comment})
- if chunked_upload:
- if offset > 0:
- pywikibot.log(f'Continuing upload from byte {offset}')
- poll = False
- while True:
-
- if poll:
- # run a poll; not possible in first iteration
- assert file_key
- req = self.site.simple_request(
- action='upload',
- token=token,
- filekey=file_key,
- checkstatus=True)
- else:
- f.seek(offset)
- chunk = f.read(self.chunk_size)
- # workaround (hack) for T132676
- # append another '\r' so that one is the payload
- # and the second is used for newline when mangled
- # by email package.
- if (len(chunk) < self.chunk_size
- or (offset + len(chunk)) == filesize
- and chunk[-1] == b'\r'[0]):
- chunk += b'\r'
-
- mime_params = {
- 'chunk': (chunk,
- ('application', 'octet-stream'),
- {'filename': mime_filename})
- }
- req = self.site._request(
- throttle=throttle,
- mime=mime_params,
- parameters={
- 'action': 'upload',
- 'token': token,
- 'stash': True,
- 'filesize': filesize,
- 'offset': offset,
- 'filename': file_page_title,
- 'async': self.asynchronous,
- 'ignorewarnings': ignore_all_warnings})
-
- if file_key:
- req['filekey'] = file_key
-
- try:
- data = req.submit()['upload']
- except APIError as error:
- # TODO: catch and process foreseeable errors
- if error.code == 'stashfailed' \
- and 'offset' in error.other:
- # TODO: Ask MediaWiki to change this
- # ambiguous error code.
-
- new_offset = int(error.other['offset'])
- # If the offset returned from the server
- # (the offset it expects now) is equal to
- # the offset we sent it, there must be
- # something else that prevented the upload,
- # instead of simple offset mismatch. This
- # also prevents infinite loops when we
- # upload the same chunk again and again,
- # every time ApiError.
- if offset != new_offset:
- pywikibot.log(
- f'Old offset: {offset}; Returned '
- f'offset: {new_offset}; Chunk size: '
- f'{len(chunk)}'
- )
- pywikibot.warning('Attempting to correct '
- 'automatically from '
- 'offset mismatch error.')
- offset = new_offset
- continue
- raise
- if 'nochange' in data: # in simulation mode
- break
-
- # Polls may not contain file key in response
- file_key = data.get('filekey', file_key)
- if data['result'] == 'Warning':
- assert ('warnings' in data
- and not ignore_all_warnings)
- if callable(ignore_warnings):
- restart = False
- if 'offset' not in data:
- # This is a result of a warning in the
- # first chunk. The chunk is not actually
- # stashed so upload must be restarted if
- # the warning is allowed.
- # T112416 and T112405#1637544
- restart = True
- data['offset'] = True
- if ignore_warnings(self.create_warnings_list(
- data, file_key)):
- # Future warnings of this run
- # can be ignored
- if restart:
- return self._upload(
- ignore_warnings=True,
- report_success=False
- )
-
- ignore_warnings = True
- ignore_all_warnings = True
- offset = data['offset']
- continue
- return False
- result = data
- result.setdefault('offset', 0)
- break
-
- if data['result'] == 'Continue':
- throttle = False
- if 'offset' in data:
- new_offset = int(data['offset'])
- if offset + len(chunk) != new_offset:
- pywikibot.log(
- f'Old offset: {offset}; Returned '
- f'offset: {new_offset}; Chunk size: '
- f'{len(chunk)}'
- )
- pywikibot.warning('Unexpected offset.')
- offset = new_offset
- else:
- pywikibot.warning('Offset was not supplied.')
- offset += len(chunk)
- elif data['result'] == 'Poll':
- poll = True
- pywikibot.log('Waiting for server to '
- 'assemble chunks.')
- elif data['result'] == 'Success': # finished
- pywikibot.log('Finished uploading last chunk.')
- final_request['filekey'] = file_key
- final_request['async'] = self.asynchronous
- break
- else:
- raise Error('Unrecognized result: {result}'
- .format_map(data))
-
- elif file_key:
- final_request['filekey'] = file_key
- else:
- file_contents = f.read()
- filetype = (mimetypes.guess_type(self.filename)[0]
- or 'application/octet-stream')
- final_request.mime = {
- 'file': (file_contents, filetype.split('/'),
- {'filename': mime_filename})
- }
+ if self.filename and state.file_key:
+ self._validate_resume_offset(state)
+ state.stage = _UploadStage.RECOVER_STASH
else:
- # upload by URL
- if not self.site.has_right('upload_by_url'):
- raise Error(f"User '{self.site.user()}' is not authorized to "
- f'upload by URL on site {self}.')
- final_request = self.site.simple_request(
- action='upload', filename=file_page_title, url=self.url,
- comment=self.comment, text=self.text, token=token)
+ self._prepare_request(state)
- return self.submit(final_request, result, data.get('result'),
- ignore_warnings, ignore_all_warnings,
- report_success, file_key)
+ def _validate_resume_offset(self, state: _UploadState) -> None:
+ """Validate a client-provided resume offset."""
+ assert (state.resume_mode is _ResumeMode.COMPLETE_STASH
+ or state.file_size is not None)
+ file_size = state.file_size
+ if (state.resume_mode is _ResumeMode.AT_OFFSET
+ and file_size is not None
+ and state.offset > file_size):
+ raise ValueError(
+ f'For the file key "{state.file_key}" the offset was set to '
+ f'{state.offset} while the file is only '
+ f'{file_size} bytes large.'
+ )
- def submit(self, request, result, data_result: str | None,
- ignore_warnings, ignore_all_warnings, report_success,
- file_key) -> bool:
- """Submit request and return whether upload was successful."""
- # some warning keys have been changed
+ def _recover_stash(self, state: _UploadState) -> None:
+ """Validate a server stash and select its next stage."""
+ if not state.file_key:
+ raise ValueError('Without a file key it cannot request the '
+ 'stash information')
+ if not self.filename:
+ raise ValueError('Can request stash information only when '
+ 'using a file name.')
+
+ stash_info = self.site.stash_info(
+ state.file_key, ['size', 'sha1'])
+ if state.resume_mode is _ResumeMode.DISCOVER_OFFSET:
+ state.offset = stash_info['size']
+ elif state.resume_mode is _ResumeMode.COMPLETE_STASH:
+ if state.file_size != stash_info['size']:
+ raise ValueError(
+ f'For the file key "{state.file_key}" the server reported '
+ f'a size {stash_info["size"]} while the file size is '
+ f'{state.file_size}')
+ assert state.file_size is not None
+ state.offset = state.file_size
+ elif state.offset != stash_info['size']:
+ raise ValueError(
+ f'For the file key "{state.file_key}" the server reported '
+ f'a size {stash_info["size"]} while the offset was '
+ f'{state.offset}')
+
+ assert 'sha1' in stash_info, f'sha1 not in stash info: {stash_info}'
+ sha1 = compute_file_hash(
+ self.filename,
+ bytes_to_read=(
+ None if state.resume_mode is _ResumeMode.COMPLETE_STASH
+ else state.offset))
+ if sha1 != stash_info['sha1']:
+ raise ValueError(
+ f'The SHA1 of {state.offset} bytes of the stashed '
+ f'"{state.file_key}" is {stash_info["sha1"]} '
+ f'while the local file is {sha1}')
+
+ self._prepare_request(state)
+
+ def _prepare_request(self, state: _UploadState) -> None:
+ """Build an upload request or select chunk transfer."""
+ reuse_stash = (
+ bool(state.file_key)
+ and state.resume_mode is _ResumeMode.COMPLETE_STASH
+ ) or state.offset == state.file_size
+ if reuse_stash:
+ pywikibot.log(
+ 'Reused already upload file using filekey '
+ f'"{state.file_key}"')
+ # TODO: Use sessionkey instead of filekey if necessary
+ state.request = self.site._request(parameters={
+ 'action': 'upload',
+ 'token': state.token,
+ 'filename': state.file_page_title,
+ 'comment': self.comment,
+ 'text': self.text,
+ 'async': self.asynchronous,
+ 'filekey': state.file_key,
+ })
+ state.stage = _UploadStage.FINALIZE
+ return
+
+ if self.filename:
+ self._prepare_file_request(state)
+ return
+
+ if not self.site.has_right('upload_by_url'):
+ raise Error(f"User '{self.site.user()}' is not authorized to "
+ f'upload by URL on site {self}.')
+ state.request = self.site.simple_request(
+ action='upload', filename=state.file_page_title, url=self.url,
+ comment=self.comment, text=self.text, token=state.token)
+ state.stage = _UploadStage.FINALIZE
+
+ def _prepare_file_request(self, state: _UploadState) -> None:
+ """Build a local-file request and select its transfer mode."""
+ assert self.filename is not None
+ state.file_size = os.path.getsize(self.filename)
+ state.request = self.site._request(
+ throttle=state.throttle,
+ parameters={
+ 'action': 'upload',
+ 'token': state.token,
+ 'text': self.text,
+ 'filename': state.file_page_title,
+ 'comment': self.comment,
+ })
+ if 0 < self.chunk_size < state.file_size:
+ state.stage = _UploadStage.TRANSFER
+ return
+
+ if state.file_key:
+ state.request['filekey'] = state.file_key
+ else:
+ # TODO: Dummy value to allow Unicode names, see T75661
+ with open(self.filename, 'rb') as f:
+ file_contents = f.read()
+ filetype = (mimetypes.guess_type(self.filename)[0]
+ or 'application/octet-stream')
+ state.request.mime = {
+ 'file': (file_contents, filetype.split('/'),
+ {'filename': 'FAKE-NAME'})
+ }
+ state.stage = _UploadStage.FINALIZE
+
+ def _transfer_chunks(self, state: _UploadState) -> bool | None:
+ """Transfer chunks until another upload stage is required."""
+ assert self.filename is not None
+ if state.offset > 0:
+ pywikibot.log(f'Continuing upload from byte {state.offset}')
+
+ chunk = b''
+ with open(self.filename, 'rb') as f:
+ while state.stage in {
+ _UploadStage.TRANSFER,
+ _UploadStage.POLL_TRANSFER,
+ }:
+ request, new_chunk = self._next_chunk_request(state, f)
+ if new_chunk is not None:
+ chunk = new_chunk
+ try:
+ data = request.submit()['upload']
+ except APIError as error:
+ if self._correct_stash_offset(error, state, chunk):
+ continue
+ raise
+
+ outcome = self._handle_chunk_response(state, data, chunk)
+ if outcome is not None:
+ return outcome
+ return None
+
+ def _next_chunk_request(self, state: _UploadState, file_obj):
+ """Create the next chunk-transfer or polling request."""
+ if state.stage is _UploadStage.POLL_TRANSFER:
+ assert state.file_key
+ return self.site.simple_request(
+ action='upload', token=state.token,
+ filekey=state.file_key, checkstatus=True), None
+
+ file_obj.seek(state.offset)
+ chunk = file_obj.read(self.chunk_size)
+ # Workaround for T132676: append another carriage return so that
+ # one remains in the payload if the email package mangles it.
+ if (len(chunk) < self.chunk_size
+ or (state.offset + len(chunk)) == state.file_size
+ and chunk[-1] == b'\r'[0]):
+ chunk += b'\r'
+ request = self.site._request(
+ throttle=state.throttle,
+ mime={
+ 'chunk': (chunk, ('application', 'octet-stream'),
+ {'filename': 'FAKE-NAME'}),
+ },
+ parameters={
+ 'action': 'upload',
+ 'token': state.token,
+ 'stash': True,
+ 'filesize': state.file_size,
+ 'offset': state.offset,
+ 'filename': state.file_page_title,
+ 'async': self.asynchronous,
+ 'ignorewarnings': state.ignore_all_warnings,
+ })
+ if state.file_key:
+ request['filekey'] = state.file_key
+ return request, chunk
+
+ @staticmethod
+ def _correct_stash_offset(error, state: _UploadState, chunk) -> bool:
+ """Correct a mismatched stash offset when the server supplies one."""
+ if error.code != 'stashfailed' or 'offset' not in error.other:
+ return False
+ new_offset = int(error.other['offset'])
+ if state.offset == new_offset:
+ return False
+ pywikibot.log(
+ f'Old offset: {state.offset}; Returned offset: {new_offset}; '
+ f'Chunk size: {len(chunk)}')
+ pywikibot.warning('Attempting to correct automatically from '
+ 'offset mismatch error.')
+ state.offset = new_offset
+ return True
+
+ def _handle_chunk_response(self, state: _UploadState, data,
+ chunk) -> bool | None:
+ """Apply a chunk response and select the next stage."""
+ state.data_result = data.get('result')
+ if 'nochange' in data: # in simulation mode
+ state.stage = _UploadStage.FINALIZE
+ return None
+
+ # Polls may not contain a file key in the response.
+ state.file_key = data.get('filekey', state.file_key)
+ if data['result'] == 'Warning':
+ return self._handle_chunk_warning(state, data)
+ if data['result'] == 'Continue':
+ self._continue_chunk(state, data, chunk)
+ elif data['result'] == 'Poll':
+ state.stage = _UploadStage.POLL_TRANSFER
+ pywikibot.log('Waiting for server to assemble chunks.')
+ elif data['result'] == 'Success':
+ pywikibot.log('Finished uploading last chunk.')
+ state.request['filekey'] = state.file_key
+ state.request['async'] = self.asynchronous
+ state.stage = _UploadStage.FINALIZE
+ else:
+ raise Error('Unrecognized result: {result}'.format_map(data))
+ return None
+
+ def _handle_chunk_warning(self, state: _UploadState,
+ data) -> bool | None:
+ """Resolve a warning returned while transferring chunks."""
+ assert 'warnings' in data and not state.ignore_all_warnings
+ if not callable(state.ignore_warnings):
+ state.response = data
+ state.response.setdefault('offset', 0)
+ state.stage = _UploadStage.FINALIZE
+ return None
+
+ restart = 'offset' not in data
+ if restart:
+ # A first-chunk warning has no stashed data to continue.
+ # See T112416 and T112405#1637544.
+ data['offset'] = True
+ warnings = self.create_warnings_list(data, state.file_key)
+ if not state.ignore_warnings(warnings):
+ return False
+
+ self._ignore_future_warnings(state)
+ if restart:
+ self._restart_upload(state)
+ else:
+ state.offset = data['offset']
+ return None
+
+ @staticmethod
+ def _continue_chunk(state: _UploadState, data, chunk) -> None:
+ """Advance state after a successful chunk transfer."""
+ state.throttle = False
+ if 'offset' in data:
+ new_offset = int(data['offset'])
+ if state.offset + len(chunk) != new_offset:
+ pywikibot.log(
+ f'Old offset: {state.offset}; Returned offset: '
+ f'{new_offset}; Chunk size: {len(chunk)}')
+ pywikibot.warning('Unexpected offset.')
+ state.offset = new_offset
+ else:
+ pywikibot.warning('Offset was not supplied.')
+ state.offset += len(chunk)
+
+ @staticmethod
+ def _ignore_future_warnings(state: _UploadState) -> None:
+ """Ignore any warnings after the callback accepts them."""
+ state.ignore_warnings = True
+ state.ignore_all_warnings = True
+ state.report_success = False
+
+ def _restart_upload(self, state: _UploadState, file_key=None,
+ offset=0) -> None:
+ """Start a new attempt with explicit resume information."""
+ assert offset == 0 or file_key is not None
+ state.stage = _UploadStage.PREPARE
+ state.file_key = file_key
+ state.offset = 0
+ if not self.filename:
+ state.resume_mode = _ResumeMode.FRESH
+ elif offset is True:
+ state.resume_mode = _ResumeMode.DISCOVER_OFFSET
+ elif offset is False:
+ state.resume_mode = _ResumeMode.COMPLETE_STASH
+ elif file_key:
+ state.resume_mode = _ResumeMode.AT_OFFSET
+ state.offset = offset
+ else:
+ state.resume_mode = _ResumeMode.FRESH
+ state.request = None
+ state.response = None
+ state.data_result = None
+
+ def _finalize_upload(self, state: _UploadState) -> bool | None:
+ """Submit or poll the final publication request."""
+ assert state.request is not None
+ if not state.response:
+ state.request['watch'] = self.watch
+ state.request['ignorewarnings'] = state.ignore_all_warnings
+ state.response = state.request.submit()['upload']
+ pywikibot.debug(state.response)
+
+ if 'result' not in state.response:
+ raise Error(
+ f'Upload: unrecognized response: {state.response}')
+ if state.response['result'] == 'Warning':
+ return self._handle_final_warning(state)
+ if state.response['result'] == 'Poll':
+ self._poll_publication(state)
+ return None
+ if state.response['result'] == 'Success':
+ return self._complete_upload(state)
+ raise Error(
+ 'Unrecognized result: '
+ f"{state.data_result or state.response['result']}")
+
+ def _handle_final_warning(self, state: _UploadState) -> bool | None:
+ """Resolve a warning returned during final publication."""
+ result = state.response
+ assert result is not None
+ assert 'warnings' in result and not state.ignore_all_warnings
+ warnings = result['warnings']
+ self._update_warning_file_key(state, result)
+
+ if not state.report_success:
+ result.setdefault('offset', bool(self.filename))
+ offset = result['offset'] if self.filename else False
+ warning_list = self.create_warnings_list(
+ result, state.file_key)
+ if state.ignore_warnings(warning_list):
+ self._ignore_future_warnings(state)
+ self._restart_upload(state, state.file_key, offset)
+ return None
+ return False
+
+ if len(warnings) > 1:
+ warn(f'The upload returned {len(warnings)} warnings: '
+ f'{", ".join(warnings)}', UserWarning, 3)
+ warning, message = next(iter(warnings.items()))
warning_keys = {
'nochange': 'no-change',
'duplicateversions': 'duplicate-version',
'emptyfile': 'empty-file',
}
+ warning = warning_keys.get(warning, warning)
+ raise UploadError(warning,
+ self.upload_warnings[warning].format(msg=message),
+ file_key=state.file_key,
+ offset=result.get('offset', False))
- token = request['token']
- while True:
- if not result:
- request['watch'] = self.watch
- request['ignorewarnings'] = ignore_all_warnings
- result = request.submit()['upload']
- pywikibot.debug(result)
+ def _update_warning_file_key(self, state: _UploadState, result) -> None:
+ """Update the file key supplied with a final warning."""
+ if not self.filename:
+ state.file_key = None
+ elif 'filekey' in result:
+ state.file_key = result['filekey']
+ elif 'sessionkey' in result:
+ # TODO: Reflect sessionkey in the API call above if necessary.
+ state.file_key = result['sessionkey']
+ pywikibot.warning('Using sessionkey instead of filekey.')
+ else:
+ state.file_key = None
+ pywikibot.warning('No filekey defined.')
- if 'result' not in result:
- raise Error(f'Upload: unrecognized response: {result}')
+ def _poll_publication(self, state: _UploadState) -> None:
+ """Prepare a status request for asynchronous publication."""
+ assert state.file_key
+ pywikibot.log('Waiting for upload to be published.')
+ state.response = None
+ state.request = self.site.simple_request(
+ action='upload', token=state.token,
+ filekey=state.file_key, checkstatus=True)
+ state.stage = _UploadStage.POLL_PUBLICATION
- if result['result'] == 'Warning':
- assert 'warnings' in result and not ignore_all_warnings
- warnings = result['warnings']
+ def _complete_upload(self, state: _UploadState) -> bool:
+ """Record a successful upload and return its result."""
+ result = state.response
+ assert result is not None
+ if state.report_success:
+ pywikibot.info('Upload successful.')
+ if 'nochange' not in result:
+ self.filepage._load_file_revisions([result['imageinfo']])
+ return True
- if self.filename:
- if 'filekey' in result:
- file_key = result['filekey']
- elif 'sessionkey' in result:
- # TODO: Probably needs to be reflected in the API call
- # above
- file_key = result['sessionkey']
- pywikibot.warning(
- 'Using sessionkey instead of filekey.')
- else:
- file_key = None
- pywikibot.warning('No filekey defined.')
- else:
- file_key = None
+ def submit(self, request, result, data_result: str | None,
+ ignore_warnings, ignore_all_warnings, report_success,
+ file_key) -> bool:
+ """Submit request and return whether upload was successful.
- if not report_success:
- result.setdefault('offset', bool(self.filename))
- offset = result['offset'] if self.filename else False
- if ignore_warnings(self.create_warnings_list(result,
- file_key)):
- return self._upload(ignore_warnings=True,
- report_success=False,
- file_key=file_key,
- offset=offset)
- return False
-
- if len(warnings) > 1:
- warn(f'The upload returned {len(warnings)} warnings: '
- f'{", ".join(warnings)}',
- UserWarning, 3)
- warning, message = next(iter(warnings.items()))
- warning = warning_keys.get(warning, warning)
- raise UploadError(warning,
- self.upload_warnings[warning]
- .format(msg=message),
- file_key=file_key,
- offset=result.get('offset', False))
-
- if result['result'] == 'Poll':
- # Polling is meaningless without a file key
- assert file_key
- pywikibot.log('Waiting for upload to be published.')
- result = None
- request = self.site.simple_request(
- action='upload',
- token=token,
- filekey=file_key,
- checkstatus=True)
- continue
-
- if result['result'] == 'Success':
- if report_success:
- pywikibot.info('Upload successful.')
- # If we receive a nochange, that would mean we're in simulation
- # mode, don't attempt to access imageinfo
- if 'nochange' not in result:
- self.filepage._load_file_revisions([result['imageinfo']])
- return True
-
- raise Error(
- f"Unrecognized result: {data_result or result['result']}")
+ This compatibility entry point starts the state machine at final
+ publication. New upload orchestration uses private stage helpers.
+ """
+ state = _UploadState(
+ ignore_warnings,
+ ignore_all_warnings,
+ report_success,
+ stage=_UploadStage.FINALIZE,
+ file_key=file_key,
+ token=request['token'],
+ request=request,
+ response=result,
+ data_result=data_result,
+ )
+ return self._run_upload(state)
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1325981?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I95eb1f88d6a957e619c286b2a67626cf43ac343a
Gerrit-Change-Number: 1325981
Gerrit-PatchSet: 7
Gerrit-Owner: Mahveotm <[email protected]>
Gerrit-Reviewer: Multichill <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
