Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package python-google-resumable-media for
openSUSE:Factory checked in at 2023-09-13 20:45:42
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-google-resumable-media (Old)
and /work/SRC/openSUSE:Factory/.python-google-resumable-media.new.1766
(New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-google-resumable-media"
Wed Sep 13 20:45:42 2023 rev:18 rq:1110838 version:2.6.0
Changes:
--------
---
/work/SRC/openSUSE:Factory/python-google-resumable-media/python-google-resumable-media.changes
2023-05-11 14:13:58.717507158 +0200
+++
/work/SRC/openSUSE:Factory/.python-google-resumable-media.new.1766/python-google-resumable-media.changes
2023-09-13 20:47:56.765311458 +0200
@@ -1,0 +2,11 @@
+Tue Sep 12 13:11:29 UTC 2023 - John Paul Adrian Glaubitz
<[email protected]>
+
+- Update to 2.6.0
+ Features
+ * Add support for concurrent XML MPU uploads (#395)
+ * Introduce compatibility with native namespace packages (#385)
+ Bug Fixes
+ * Add google-auth to aiohttp extra (#386)
+- Update file pattern in %files section
+
+-------------------------------------------------------------------
Old:
----
google-resumable-media-2.5.0.tar.gz
New:
----
google-resumable-media-2.6.0.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-google-resumable-media.spec ++++++
--- /var/tmp/diff_new_pack.4ivLbF/_old 2023-09-13 20:47:57.797348275 +0200
+++ /var/tmp/diff_new_pack.4ivLbF/_new 2023-09-13 20:47:57.797348275 +0200
@@ -19,7 +19,7 @@
%{?!python_module:%define python_module() python-%{**} python3-%{**}}
%define skip_python2 1
Name: python-google-resumable-media
-Version: 2.5.0
+Version: 2.6.0
Release: 0
Summary: Utilities for Google Media Downloads and Resumable Uploads
License: Apache-2.0
@@ -62,7 +62,6 @@
%license LICENSE
%doc README.rst
%{python_sitelib}/google_resumable_media*-info
-%{python_sitelib}/google_resumable_media*pth
%{python_sitelib}/google/resumable_media*
%{python_sitelib}/google/_async_resumable_media*
++++++ google-resumable-media-2.5.0.tar.gz ->
google-resumable-media-2.6.0.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/google-resumable-media-2.5.0/PKG-INFO
new/google-resumable-media-2.6.0/PKG-INFO
--- old/google-resumable-media-2.5.0/PKG-INFO 2023-04-24 21:02:52.186299300
+0200
+++ new/google-resumable-media-2.6.0/PKG-INFO 2023-09-06 20:12:17.037366400
+0200
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: google-resumable-media
-Version: 2.5.0
+Version: 2.6.0
Summary: Utilities for Google Media Downloads and Resumable Uploads
Home-page: https://github.com/googleapis/google-resumable-media-python
Author: Google Cloud Platform
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/google-resumable-media-2.5.0/google/__init__.py
new/google-resumable-media-2.6.0/google/__init__.py
--- old/google-resumable-media-2.5.0/google/__init__.py 2023-04-24
21:00:19.000000000 +0200
+++ new/google-resumable-media-2.6.0/google/__init__.py 1970-01-01
01:00:00.000000000 +0100
@@ -1,22 +0,0 @@
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-try:
- import pkg_resources
-
- pkg_resources.declare_namespace(__name__)
-except ImportError:
- import pkgutil
-
- __path__ = pkgutil.extend_path(__path__, __name__) # type: ignore
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/google-resumable-media-2.5.0/google/resumable_media/_helpers.py
new/google-resumable-media-2.6.0/google/resumable_media/_helpers.py
--- old/google-resumable-media-2.5.0/google/resumable_media/_helpers.py
2023-04-24 21:00:19.000000000 +0200
+++ new/google-resumable-media-2.6.0/google/resumable_media/_helpers.py
2023-09-06 20:09:24.000000000 +0200
@@ -243,6 +243,34 @@
return (expected_checksum, checksum_object)
+def _get_uploaded_checksum_from_headers(response, get_headers, checksum_type):
+ """Get the computed checksum and checksum object from the response headers.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+ get_headers (callable: response->dict): returns response headers.
+ checksum_type Optional(str): The checksum type to read from the
headers,
+ exactly as it will appear in the headers (case-sensitive). Must be
+ "md5", "crc32c" or None.
+
+ Returns:
+ Tuple (Optional[str], object): The checksum of the response,
+ if it can be detected from the ``X-Goog-Hash`` header, and the
+ appropriate checksum object for the expected checksum.
+ """
+ if checksum_type not in ["md5", "crc32c", None]:
+ raise ValueError("checksum must be ``'md5'``, ``'crc32c'`` or
``None``")
+ elif checksum_type in ["md5", "crc32c"]:
+ headers = get_headers(response)
+ remote_checksum = _parse_checksum_header(
+ headers.get(_HASH_HEADER), response, checksum_label=checksum_type
+ )
+ else:
+ remote_checksum = None
+
+ return remote_checksum
+
+
def _parse_checksum_header(header_value, response, checksum_label):
"""Parses the checksum header from an ``X-Goog-Hash`` value.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/google-resumable-media-2.5.0/google/resumable_media/_upload.py
new/google-resumable-media-2.6.0/google/resumable_media/_upload.py
--- old/google-resumable-media-2.5.0/google/resumable_media/_upload.py
2023-04-24 21:00:19.000000000 +0200
+++ new/google-resumable-media-2.6.0/google/resumable_media/_upload.py
2023-09-06 20:09:24.000000000 +0200
@@ -33,6 +33,8 @@
from google.resumable_media import _helpers
from google.resumable_media import common
+from xml.etree import ElementTree
+
_CONTENT_TYPE_HEADER = "content-type"
_CONTENT_RANGE_TEMPLATE = "bytes {:d}-{:d}/{:d}"
@@ -54,6 +56,7 @@
"{:d} bytes have been read from the stream, which exceeds "
"the expected total {:d}."
)
+_DELETE = "DELETE"
_POST = "POST"
_PUT = "PUT"
_UPLOAD_CHECKSUM_MISMATCH_MESSAGE = (
@@ -63,6 +66,14 @@
_UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE = (
"Response metadata had no ``{}`` value; checksum could not be validated."
)
+_UPLOAD_HEADER_NO_APPROPRIATE_CHECKSUM_MESSAGE = (
+ "Response headers had no ``{}`` value; checksum could not be validated."
+)
+_MPU_INITIATE_QUERY = "?uploads"
+_MPU_PART_QUERY_TEMPLATE = "?partNumber={part}&uploadId={upload_id}"
+_S3_COMPAT_XML_NAMESPACE = "{http://s3.amazonaws.com/doc/2006-03-01/}";
+_UPLOAD_ID_NODE = "UploadId"
+_MPU_FINAL_QUERY_TEMPLATE = "?uploadId={upload_id}"
class UploadBase(object):
@@ -236,7 +247,7 @@
upload_url (str): The URL where the content will be uploaded.
headers (Optional[Mapping[str, str]]): Extra headers that should
be sent with the request, e.g. headers for encrypted data.
- checksum Optional([str]): The type of checksum to compute to verify
+ checksum (Optional([str])): The type of checksum to compute to verify
the integrity of the object. The request metadata will be amended
to include the computed value. Using this option will override a
manually-set checksum value. Supported values are "md5", "crc32c"
@@ -341,10 +352,8 @@
upload_url (str): The URL where the resumable upload will be initiated.
chunk_size (int): The size of each chunk used to upload the resource.
headers (Optional[Mapping[str, str]]): Extra headers that should
- be sent with the :meth:`initiate` request, e.g. headers for
- encrypted data. These **will not** be sent with
- :meth:`transmit_next_chunk` or :meth:`recover` requests.
- checksum Optional([str]): The type of checksum to compute to verify
+ be sent with every request.
+ checksum (Optional([str])): The type of checksum to compute to verify
the integrity of the object. After the upload is complete, the
server-computed checksum of the resulting object will be read
and google.resumable_media.common.DataCorruption will be raised on
@@ -587,8 +596,7 @@
* the body of the request
* headers for the request
- The headers **do not** incorporate the ``_headers`` on the
- current instance.
+ The headers incorporate the ``_headers`` on the current instance.
Raises:
ValueError: If the current upload has finished.
@@ -718,7 +726,7 @@
self._bytes_uploaded = int(match.group("end_byte")) + 1
def _validate_checksum(self, response):
- """Check the computed checksum, if any, against the response headers.
+ """Check the computed checksum, if any, against the recieved metadata.
Args:
response (object): The HTTP response object.
@@ -860,6 +868,512 @@
raise NotImplementedError("This implementation is virtual.")
+class XMLMPUContainer(UploadBase):
+ """Initiate and close an upload using the XML MPU API.
+
+ An XML MPU sends an initial request and then receives an upload ID.
+ Using the upload ID, the upload is then done in numbered parts and the
+ parts can be uploaded concurrently.
+
+ In order to avoid concurrency issues with this container object, the
+ uploading of individual parts is handled separately, by XMLMPUPart objects
+ spawned from this container class. The XMLMPUPart objects are not
+ necessarily in the same process as the container, so they do not update the
+ container automatically.
+
+ MPUs are sometimes referred to as "Multipart Uploads", which is ambiguous
+ given the JSON multipart upload, so the abbreviation "MPU" will be used
+ throughout.
+
+ See: https://cloud.google.com/storage/docs/multipart-uploads
+
+ Args:
+ upload_url (str): The URL of the object (without query parameters). The
+ initiate, PUT, and finalization requests will all use this URL,
with
+ varying query parameters.
+ filename (str): The name (path) of the file to upload.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with every request.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+ upload_id (Optional(str)): The ID of the upload from the initialization
+ response.
+ """
+
+ def __init__(self, upload_url, filename, headers=None, upload_id=None):
+ super().__init__(upload_url, headers=headers)
+ self._filename = filename
+ self._upload_id = upload_id
+ self._parts = {}
+
+ @property
+ def upload_id(self):
+ return self._upload_id
+
+ def register_part(self, part_number, etag):
+ """Register an uploaded part by part number and corresponding etag.
+
+ XMLMPUPart objects represent individual parts, and their part number
+ and etag can be registered to the container object with this method
+ and therefore incorporated in the finalize() call to finish the upload.
+
+ This method accepts part_number and etag, but not XMLMPUPart objects
+ themselves, to reduce the complexity involved in running XMLMPUPart
+ uploads in separate processes.
+
+ Args:
+ part_number (int): The part number. Parts are assembled into the
+ final uploaded object with finalize() in order of their part
+ numbers.
+ etag (str): The etag included in the server response after upload.
+ """
+ self._parts[part_number] = etag
+
+ def _prepare_initiate_request(self, content_type):
+ """Prepare the contents of HTTP request to initiate upload.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always POST)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ Raises:
+ ValueError: If the current upload has already been initiated.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.upload_id is not None:
+ raise ValueError("This upload has already been initiated.")
+
+ initiate_url = self.upload_url + _MPU_INITIATE_QUERY
+
+ headers = {
+ **self._headers,
+ _CONTENT_TYPE_HEADER: content_type,
+ }
+ return _POST, initiate_url, None, headers
+
+ def _process_initiate_response(self, response):
+ """Process the response from an HTTP request that initiated the upload.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ This method takes the URL from the ``Location`` header and stores it
+ for future use. Within that URL, we assume the ``upload_id`` query
+ parameter has been included, but we do not check.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 200.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ _helpers.require_status_code(response, (http.client.OK,),
self._get_status_code)
+ root = ElementTree.fromstring(response.text)
+ self._upload_id = root.find(_S3_COMPAT_XML_NAMESPACE +
_UPLOAD_ID_NODE).text
+
+ def initiate(
+ self,
+ transport,
+ content_type,
+ timeout=None,
+ ):
+ """Initiate an MPU and record the upload ID.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ def _prepare_finalize_request(self):
+ """Prepare the contents of an HTTP request to finalize the upload.
+
+ All of the parts must be registered before calling this method.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always POST)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ Raises:
+ ValueError: If the upload has not been initiated.
+ """
+ if self.upload_id is None:
+ raise ValueError("This upload has not yet been initiated.")
+
+ final_query =
_MPU_FINAL_QUERY_TEMPLATE.format(upload_id=self._upload_id)
+ finalize_url = self.upload_url + final_query
+ final_xml_root = ElementTree.Element("CompleteMultipartUpload")
+ for part_number, etag in self._parts.items():
+ part = ElementTree.SubElement(final_xml_root, "Part") # put in a
loop
+ ElementTree.SubElement(part, "PartNumber").text = str(part_number)
+ ElementTree.SubElement(part, "ETag").text = etag
+ payload = ElementTree.tostring(final_xml_root)
+ return _POST, finalize_url, payload, self._headers
+
+ def _process_finalize_response(self, response):
+ """Process the response from an HTTP request that finalized the upload.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 200.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+
+ _helpers.require_status_code(response, (http.client.OK,),
self._get_status_code)
+ self._finished = True
+
+ def finalize(
+ self,
+ transport,
+ timeout=None,
+ ):
+ """Finalize an MPU request with all the parts.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ def _prepare_cancel_request(self):
+ """Prepare the contents of an HTTP request to cancel the upload.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always DELETE)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ Raises:
+ ValueError: If the upload has not been initiated.
+ """
+ if self.upload_id is None:
+ raise ValueError("This upload has not yet been initiated.")
+
+ cancel_query =
_MPU_FINAL_QUERY_TEMPLATE.format(upload_id=self._upload_id)
+ cancel_url = self.upload_url + cancel_query
+ return _DELETE, cancel_url, None, self._headers
+
+ def _process_cancel_response(self, response):
+ """Process the response from an HTTP request that canceled the upload.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 204.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+
+ _helpers.require_status_code(
+ response, (http.client.NO_CONTENT,), self._get_status_code
+ )
+
+ def cancel(
+ self,
+ transport,
+ timeout=None,
+ ):
+ """Cancel an MPU request and permanently delete any uploaded parts.
+
+ This cannot be undone.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+class XMLMPUPart(UploadBase):
+ """Upload a single part of an existing XML MPU container.
+
+ An XML MPU sends an initial request and then receives an upload ID.
+ Using the upload ID, the upload is then done in numbered parts and the
+ parts can be uploaded concurrently.
+
+ In order to avoid concurrency issues with the container object, the
+ uploading of individual parts is handled separately by multiple objects
+ of this class. Once a part is uploaded, it can be registered with the
+ container with `container.register_part(part.part_number, part.etag)`.
+
+ MPUs are sometimes referred to as "Multipart Uploads", which is ambiguous
+ given the JSON multipart upload, so the abbreviation "MPU" will be used
+ throughout.
+
+ See: https://cloud.google.com/storage/docs/multipart-uploads
+
+ Args:
+ upload_url (str): The URL of the object (without query parameters).
+ upload_id (str): The ID of the upload from the initialization response.
+ filename (str): The name (path) of the file to upload.
+ start (int): The byte index of the beginning of the part.
+ end (int): The byte index of the end of the part.
+ part_number (int): The part number. Part numbers will be assembled in
+ sequential order when the container is finalized.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with every request.
+ checksum (Optional([str])): The type of checksum to compute to verify
+ the integrity of the object. The request headers will be amended
+ to include the computed value. Supported values are "md5", "crc32c"
+ and None. The default is None.
+
+ Attributes:
+ upload_url (str): The URL of the object (without query parameters).
+ upload_id (str): The ID of the upload from the initialization response.
+ filename (str): The name (path) of the file to upload.
+ start (int): The byte index of the beginning of the part.
+ end (int): The byte index of the end of the part.
+ part_number (int): The part number. Part numbers will be assembled in
+ sequential order when the container is finalized.
+ etag (Optional(str)): The etag returned by the service after upload.
+ """
+
+ def __init__(
+ self,
+ upload_url,
+ upload_id,
+ filename,
+ start,
+ end,
+ part_number,
+ headers=None,
+ checksum=None,
+ ):
+ super().__init__(upload_url, headers=headers)
+ self._filename = filename
+ self._start = start
+ self._end = end
+ self._upload_id = upload_id
+ self._part_number = part_number
+ self._etag = None
+ self._checksum_type = checksum
+ self._checksum_object = None
+
+ @property
+ def part_number(self):
+ return self._part_number
+
+ @property
+ def upload_id(self):
+ return self._upload_id
+
+ @property
+ def filename(self):
+ return self._filename
+
+ @property
+ def etag(self):
+ return self._etag
+
+ @property
+ def start(self):
+ return self._start
+
+ @property
+ def end(self):
+ return self._end
+
+ def _prepare_upload_request(self):
+ """Prepare the contents of HTTP request to upload a part.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O. This is based on the `sans-I/O`_ philosophy.
+
+ For the time being, this **does require** some form of I/O to read
+ a part from ``stream`` (via :func:`get_part_payload`). However, this
+ will (almost) certainly not be network I/O.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always PUT)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ The headers incorporate the ``_headers`` on the current instance.
+
+ Raises:
+ ValueError: If the current upload has finished.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.finished:
+ raise ValueError("This part has already been uploaded.")
+
+ with open(self._filename, "br") as f:
+ f.seek(self._start)
+ payload = f.read(self._end - self._start)
+
+ self._checksum_object =
_helpers._get_checksum_object(self._checksum_type)
+ if self._checksum_object is not None:
+ self._checksum_object.update(payload)
+
+ part_query = _MPU_PART_QUERY_TEMPLATE.format(
+ part=self._part_number, upload_id=self._upload_id
+ )
+ upload_url = self.upload_url + part_query
+ return _PUT, upload_url, payload, self._headers
+
+ def _process_upload_response(self, response):
+ """Process the response from an HTTP request.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 200 or the response is missing data.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ _helpers.require_status_code(
+ response,
+ (http.client.OK,),
+ self._get_status_code,
+ )
+
+ self._validate_checksum(response)
+
+ etag = _helpers.header_required(response, "etag", self._get_headers)
+ self._etag = etag
+ self._finished = True
+
+ def upload(
+ self,
+ transport,
+ timeout=None,
+ ):
+ """Upload the part.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ def _validate_checksum(self, response):
+ """Check the computed checksum, if any, against the response headers.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.DataCorruption: If the checksum
+ computed locally and the checksum reported by the remote host do
+ not match.
+ """
+ if self._checksum_type is None:
+ return
+
+ remote_checksum = _helpers._get_uploaded_checksum_from_headers(
+ response, self._get_headers, self._checksum_type
+ )
+
+ if remote_checksum is None:
+ metadata_key = _helpers._get_metadata_key(self._checksum_type)
+ raise common.InvalidResponse(
+ response,
+
_UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE.format(metadata_key),
+ self._get_headers(response),
+ )
+ local_checksum = _helpers.prepare_checksum_digest(
+ self._checksum_object.digest()
+ )
+ if local_checksum != remote_checksum:
+ raise common.DataCorruption(
+ response,
+ _UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format(
+ self._checksum_type.upper(), local_checksum,
remote_checksum
+ ),
+ )
+
+
def get_boundary():
"""Get a random boundary for a multipart request.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/google-resumable-media-2.5.0/google/resumable_media/requests/__init__.py
new/google-resumable-media-2.6.0/google/resumable_media/requests/__init__.py
---
old/google-resumable-media-2.5.0/google/resumable_media/requests/__init__.py
2023-04-24 21:00:19.000000000 +0200
+++
new/google-resumable-media-2.6.0/google/resumable_media/requests/__init__.py
2023-09-06 20:09:24.000000000 +0200
@@ -669,7 +669,8 @@
from google.resumable_media.requests.download import RawDownload
from google.resumable_media.requests.upload import ResumableUpload
from google.resumable_media.requests.upload import SimpleUpload
-
+from google.resumable_media.requests.upload import XMLMPUContainer
+from google.resumable_media.requests.upload import XMLMPUPart
__all__ = [
"ChunkedDownload",
@@ -679,4 +680,6 @@
"RawDownload",
"ResumableUpload",
"SimpleUpload",
+ "XMLMPUContainer",
+ "XMLMPUPart",
]
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/google-resumable-media-2.5.0/google/resumable_media/requests/upload.py
new/google-resumable-media-2.6.0/google/resumable_media/requests/upload.py
--- old/google-resumable-media-2.5.0/google/resumable_media/requests/upload.py
2023-04-24 21:00:19.000000000 +0200
+++ new/google-resumable-media-2.6.0/google/resumable_media/requests/upload.py
2023-09-06 20:09:24.000000000 +0200
@@ -555,3 +555,208 @@
return _request_helpers.wait_and_retry(
retriable_request, self._get_status_code, self._retry_strategy
)
+
+
+class XMLMPUContainer(_request_helpers.RequestsMixin, _upload.XMLMPUContainer):
+ """Initiate and close an upload using the XML MPU API.
+
+ An XML MPU sends an initial request and then receives an upload ID.
+ Using the upload ID, the upload is then done in numbered parts and the
+ parts can be uploaded concurrently.
+
+ In order to avoid concurrency issues with this container object, the
+ uploading of individual parts is handled separately, by XMLMPUPart objects
+ spawned from this container class. The XMLMPUPart objects are not
+ necessarily in the same process as the container, so they do not update the
+ container automatically.
+
+ MPUs are sometimes referred to as "Multipart Uploads", which is ambiguous
+ given the JSON multipart upload, so the abbreviation "MPU" will be used
+ throughout.
+
+ See: https://cloud.google.com/storage/docs/multipart-uploads
+
+ Args:
+ upload_url (str): The URL of the object (without query parameters). The
+ initiate, PUT, and finalization requests will all use this URL,
with
+ varying query parameters.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the :meth:`initiate` request, e.g. headers for
+ encrypted data. These headers will be propagated to individual
+ XMLMPUPart objects spawned from this container as well.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+ upload_id (Optional(int)): The ID of the upload from the initialization
+ response.
+ """
+
+ def initiate(
+ self,
+ transport,
+ content_type,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Initiate an MPU and record the upload ID.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+
+ method, url, payload, headers = self._prepare_initiate_request(
+ content_type,
+ )
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ result = transport.request(
+ method, url, data=payload, headers=headers, timeout=timeout
+ )
+
+ self._process_initiate_response(result)
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+ def finalize(
+ self,
+ transport,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Finalize an MPU request with all the parts.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+ method, url, payload, headers = self._prepare_finalize_request()
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ result = transport.request(
+ method, url, data=payload, headers=headers, timeout=timeout
+ )
+
+ self._process_finalize_response(result)
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+ def cancel(
+ self,
+ transport,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Cancel an MPU request and permanently delete any uploaded parts.
+
+ This cannot be undone.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+ method, url, payload, headers = self._prepare_cancel_request()
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ result = transport.request(
+ method, url, data=payload, headers=headers, timeout=timeout
+ )
+
+ self._process_cancel_response(result)
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+
+class XMLMPUPart(_request_helpers.RequestsMixin, _upload.XMLMPUPart):
+ def upload(
+ self,
+ transport,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Upload the part.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+ method, url, payload, headers = self._prepare_upload_request()
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ result = transport.request(
+ method, url, data=payload, headers=headers, timeout=timeout
+ )
+
+ self._process_upload_response(result)
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/google-resumable-media-2.5.0/google_resumable_media.egg-info/PKG-INFO
new/google-resumable-media-2.6.0/google_resumable_media.egg-info/PKG-INFO
--- old/google-resumable-media-2.5.0/google_resumable_media.egg-info/PKG-INFO
2023-04-24 21:02:52.000000000 +0200
+++ new/google-resumable-media-2.6.0/google_resumable_media.egg-info/PKG-INFO
2023-09-06 20:12:16.000000000 +0200
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: google-resumable-media
-Version: 2.5.0
+Version: 2.6.0
Summary: Utilities for Google Media Downloads and Resumable Uploads
Home-page: https://github.com/googleapis/google-resumable-media-python
Author: Google Cloud Platform
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/google-resumable-media-2.5.0/google_resumable_media.egg-info/SOURCES.txt
new/google-resumable-media-2.6.0/google_resumable_media.egg-info/SOURCES.txt
---
old/google-resumable-media-2.5.0/google_resumable_media.egg-info/SOURCES.txt
2023-04-24 21:02:52.000000000 +0200
+++
new/google-resumable-media-2.6.0/google_resumable_media.egg-info/SOURCES.txt
2023-09-06 20:12:16.000000000 +0200
@@ -3,7 +3,6 @@
README.rst
setup.cfg
setup.py
-google/__init__.py
google/_async_resumable_media/__init__.py
google/_async_resumable_media/_download.py
google/_async_resumable_media/_helpers.py
@@ -24,7 +23,6 @@
google_resumable_media.egg-info/PKG-INFO
google_resumable_media.egg-info/SOURCES.txt
google_resumable_media.egg-info/dependency_links.txt
-google_resumable_media.egg-info/namespace_packages.txt
google_resumable_media.egg-info/not-zip-safe
google_resumable_media.egg-info/requires.txt
google_resumable_media.egg-info/top_level.txt
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/google-resumable-media-2.5.0/google_resumable_media.egg-info/namespace_packages.txt
new/google-resumable-media-2.6.0/google_resumable_media.egg-info/namespace_packages.txt
---
old/google-resumable-media-2.5.0/google_resumable_media.egg-info/namespace_packages.txt
2023-04-24 21:02:52.000000000 +0200
+++
new/google-resumable-media-2.6.0/google_resumable_media.egg-info/namespace_packages.txt
1970-01-01 01:00:00.000000000 +0100
@@ -1 +0,0 @@
-google
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/google-resumable-media-2.5.0/google_resumable_media.egg-info/requires.txt
new/google-resumable-media-2.6.0/google_resumable_media.egg-info/requires.txt
---
old/google-resumable-media-2.5.0/google_resumable_media.egg-info/requires.txt
2023-04-24 21:02:52.000000000 +0200
+++
new/google-resumable-media-2.6.0/google_resumable_media.egg-info/requires.txt
2023-09-06 20:12:16.000000000 +0200
@@ -2,6 +2,7 @@
[aiohttp]
aiohttp<4.0.0dev,>=3.6.2
+google-auth<2.0dev,>=1.22.0
[requests]
requests<3.0.0dev,>=2.18.0
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/google-resumable-media-2.5.0/google_resumable_media.egg-info/top_level.txt
new/google-resumable-media-2.6.0/google_resumable_media.egg-info/top_level.txt
---
old/google-resumable-media-2.5.0/google_resumable_media.egg-info/top_level.txt
2023-04-24 21:02:52.000000000 +0200
+++
new/google-resumable-media-2.6.0/google_resumable_media.egg-info/top_level.txt
2023-09-06 20:12:16.000000000 +0200
@@ -1 +1,2 @@
google
+testing
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/google-resumable-media-2.5.0/setup.py
new/google-resumable-media-2.6.0/setup.py
--- old/google-resumable-media-2.5.0/setup.py 2023-04-24 21:00:19.000000000
+0200
+++ new/google-resumable-media-2.6.0/setup.py 2023-09-06 20:09:24.000000000
+0200
@@ -30,20 +30,21 @@
'requests': [
'requests >= 2.18.0, < 3.0.0dev',
],
- 'aiohttp': 'aiohttp >= 3.6.2, < 4.0.0dev'
+ 'aiohttp': ['aiohttp >= 3.6.2, < 4.0.0dev', 'google-auth >= 1.22.0, <
2.0dev']
}
setuptools.setup(
name='google-resumable-media',
- version = "2.5.0",
+ version = "2.6.0",
description='Utilities for Google Media Downloads and Resumable Uploads',
author='Google Cloud Platform',
author_email='[email protected]',
long_description=README,
- namespace_packages=['google'],
scripts=[],
url='https://github.com/googleapis/google-resumable-media-python',
- packages=setuptools.find_packages(exclude=('tests*',)),
+ packages=setuptools.find_namespace_packages(
+ exclude=("tests*", "docs*")
+ ),
license='Apache 2.0',
platforms='Posix; MacOS X; Windows',
include_package_data=True,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/google-resumable-media-2.5.0/tests/system/requests/conftest.py
new/google-resumable-media-2.6.0/tests/system/requests/conftest.py
--- old/google-resumable-media-2.5.0/tests/system/requests/conftest.py
2023-04-24 21:00:19.000000000 +0200
+++ new/google-resumable-media-2.6.0/tests/system/requests/conftest.py
2023-09-06 20:09:24.000000000 +0200
@@ -53,6 +53,6 @@
def bucket(authorized_transport):
ensure_bucket(authorized_transport)
- yield utils.BUCKET_URL
+ yield utils.BUCKET_NAME
cleanup_bucket(authorized_transport)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/google-resumable-media-2.5.0/tests/system/requests/test_upload.py
new/google-resumable-media-2.6.0/tests/system/requests/test_upload.py
--- old/google-resumable-media-2.5.0/tests/system/requests/test_upload.py
2023-04-24 21:00:19.000000000 +0200
+++ new/google-resumable-media-2.6.0/tests/system/requests/test_upload.py
2023-09-06 20:09:24.000000000 +0200
@@ -659,3 +659,118 @@
content_type=BYTES_CONTENT_TYPE,
)
self._check_range_sent(response2, 2 * chunk_size, total_bytes - 1,
total_bytes)
+
+
[email protected]("checksum", ["md5", "crc32c", None])
+def test_XMLMPU(authorized_transport, bucket, cleanup, checksum):
+ with open(ICO_FILE, "rb") as file_obj:
+ actual_contents = file_obj.read()
+
+ blob_name = os.path.basename(ICO_FILE)
+ # Make sure to clean up the uploaded blob when we are done.
+ cleanup(blob_name, authorized_transport)
+ check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.XML_UPLOAD_URL_TEMPLATE.format(bucket=bucket,
blob=blob_name)
+ container = resumable_requests.XMLMPUContainer(upload_url, blob_name)
+ # Initiate
+ container.initiate(authorized_transport, ICO_CONTENT_TYPE)
+ assert container.upload_id
+
+ part = resumable_requests.XMLMPUPart(
+ upload_url,
+ container.upload_id,
+ ICO_FILE,
+ 0,
+ len(actual_contents),
+ 1,
+ checksum=checksum,
+ )
+ part.upload(authorized_transport)
+ assert part.etag
+
+ container.register_part(1, part.etag)
+ container.finalize(authorized_transport)
+ assert container.finished
+
+ # Download the content to make sure it's "working as expected".
+ check_content(blob_name, actual_contents, authorized_transport)
+
+
[email protected]("checksum", ["md5", "crc32c"])
+def test_XMLMPU_with_bad_checksum(authorized_transport, bucket, checksum):
+ with open(ICO_FILE, "rb") as file_obj:
+ actual_contents = file_obj.read()
+
+ blob_name = os.path.basename(ICO_FILE)
+ # No need to clean up, since the upload will not be finalized successfully.
+ check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.XML_UPLOAD_URL_TEMPLATE.format(bucket=bucket,
blob=blob_name)
+ container = resumable_requests.XMLMPUContainer(upload_url, blob_name)
+ # Initiate
+ container.initiate(authorized_transport, ICO_CONTENT_TYPE)
+ assert container.upload_id
+
+ try:
+ part = resumable_requests.XMLMPUPart(
+ upload_url,
+ container.upload_id,
+ ICO_FILE,
+ 0,
+ len(actual_contents),
+ 1,
+ checksum=checksum,
+ )
+
+ fake_checksum_object = _helpers._get_checksum_object(checksum)
+ fake_checksum_object.update(b"bad data")
+ fake_prepared_checksum_digest = _helpers.prepare_checksum_digest(
+ fake_checksum_object.digest()
+ )
+ with mock.patch.object(
+ _helpers,
+ "prepare_checksum_digest",
+ return_value=fake_prepared_checksum_digest,
+ ):
+ with pytest.raises(common.DataCorruption):
+ part.upload(authorized_transport)
+ finally:
+ utils.retry_transient_errors(authorized_transport.delete)(
+ upload_url + "?uploadId=" + str(container.upload_id)
+ )
+
+
+def test_XMLMPU_cancel(authorized_transport, bucket):
+ with open(ICO_FILE, "rb") as file_obj:
+ actual_contents = file_obj.read()
+
+ blob_name = os.path.basename(ICO_FILE)
+ check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.XML_UPLOAD_URL_TEMPLATE.format(bucket=bucket,
blob=blob_name)
+ container = resumable_requests.XMLMPUContainer(upload_url, blob_name)
+ # Initiate
+ container.initiate(authorized_transport, ICO_CONTENT_TYPE)
+ assert container.upload_id
+
+ part = resumable_requests.XMLMPUPart(
+ upload_url,
+ container.upload_id,
+ ICO_FILE,
+ 0,
+ len(actual_contents),
+ 1,
+ )
+ part.upload(authorized_transport)
+ assert part.etag
+
+ container.register_part(1, part.etag)
+ container.cancel(authorized_transport)
+
+ # Validate the cancel worked by expecting a 404 on finalize.
+ with pytest.raises(resumable_media.InvalidResponse):
+ container.finalize(authorized_transport)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/google-resumable-media-2.5.0/tests/system/utils.py
new/google-resumable-media-2.6.0/tests/system/utils.py
--- old/google-resumable-media-2.5.0/tests/system/utils.py 2023-04-24
21:00:19.000000000 +0200
+++ new/google-resumable-media-2.6.0/tests/system/utils.py 2023-09-06
20:09:24.000000000 +0200
@@ -38,6 +38,9 @@
METADATA_URL_TEMPLATE = BUCKET_URL + "/o/{blob_name}"
+XML_UPLOAD_URL_TEMPLATE = "https://{bucket}.storage.googleapis.com/{blob}";
+
+
GCS_RW_SCOPE = "https://www.googleapis.com/auth/devstorage.read_write";
# Generated using random.choice() with all 256 byte choices.
ENCRYPTION_KEY = (
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/google-resumable-media-2.5.0/tests/unit/requests/test_upload.py
new/google-resumable-media-2.6.0/tests/unit/requests/test_upload.py
--- old/google-resumable-media-2.5.0/tests/unit/requests/test_upload.py
2023-04-24 21:00:19.000000000 +0200
+++ new/google-resumable-media-2.6.0/tests/unit/requests/test_upload.py
2023-09-06 20:09:24.000000000 +0200
@@ -15,7 +15,8 @@
import http.client
import io
import json
-
+import pytest # type: ignore
+import tempfile
from unittest import mock
import google.resumable_media.requests.upload as upload_mod
@@ -30,6 +31,25 @@
JSON_TYPE = "application/json; charset=UTF-8"
JSON_TYPE_LINE = b"content-type: application/json; charset=UTF-8\r\n"
EXPECTED_TIMEOUT = (61, 60)
+EXAMPLE_XML_UPLOAD_URL =
"https://test-project.storage.googleapis.com/test-bucket";
+EXAMPLE_XML_MPU_INITIATE_TEXT_TEMPLATE = """<?xml version="1.0"
encoding="UTF-8"?>
+<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/";>
+ <Bucket>travel-maps</Bucket>
+ <Key>paris.jpg</Key>
+ <UploadId>{upload_id}</UploadId>
+</InitiateMultipartUploadResult>
+"""
+UPLOAD_ID = "VXBsb2FkIElEIGZvciBlbHZpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"
+PARTS = {1: "39a59594290b0f9a30662a56d695b71d", 2:
"00000000290b0f9a30662a56d695b71d"}
+FILE_DATA = b"testdata" * 128
+
+
[email protected](scope="session")
+def filename():
+ with tempfile.NamedTemporaryFile() as f:
+ f.write(FILE_DATA)
+ f.flush()
+ yield f.name
class TestSimpleUpload(object):
@@ -333,8 +353,54 @@
)
-def _make_response(status_code=http.client.OK, headers=None):
+def test_mpu_container():
+ container = upload_mod.XMLMPUContainer(EXAMPLE_XML_UPLOAD_URL, filename)
+
+ response_text =
EXAMPLE_XML_MPU_INITIATE_TEXT_TEMPLATE.format(upload_id=UPLOAD_ID)
+
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _make_response(text=response_text)
+ container.initiate(transport, BASIC_CONTENT)
+ assert container.upload_id == UPLOAD_ID
+
+ for part, etag in PARTS.items():
+ container.register_part(part, etag)
+
+ assert container._parts == PARTS
+
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _make_response()
+ container.finalize(transport)
+ assert container.finished
+
+
+def test_mpu_container_cancel():
+ container = upload_mod.XMLMPUContainer(
+ EXAMPLE_XML_UPLOAD_URL, filename, upload_id=UPLOAD_ID
+ )
+
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _make_response(status_code=204)
+ container.cancel(transport)
+
+
+def test_mpu_part(filename):
+ part = upload_mod.XMLMPUPart(EXAMPLE_XML_UPLOAD_URL, UPLOAD_ID, filename,
0, 128, 1)
+
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _make_response(headers={"etag": PARTS[1]})
+
+ part.upload(transport)
+
+ assert part.finished
+ assert part.etag == PARTS[1]
+
+
+def _make_response(status_code=http.client.OK, headers=None, text=None):
headers = headers or {}
return mock.Mock(
- headers=headers, status_code=status_code, spec=["headers",
"status_code"]
+ headers=headers,
+ status_code=status_code,
+ text=text,
+ spec=["headers", "status_code", "text"],
)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/google-resumable-media-2.5.0/tests/unit/test__helpers.py
new/google-resumable-media-2.6.0/tests/unit/test__helpers.py
--- old/google-resumable-media-2.5.0/tests/unit/test__helpers.py
2023-04-24 21:00:19.000000000 +0200
+++ new/google-resumable-media-2.6.0/tests/unit/test__helpers.py
2023-09-06 20:09:24.000000000 +0200
@@ -493,6 +493,14 @@
assert new_url == "{}&{}".format(MEDIA_URL, expected)
+def test__get_uploaded_checksum_from_headers_error_handling():
+ response = _mock_response({})
+
+ with pytest.raises(ValueError):
+ _helpers._get_uploaded_checksum_from_headers(response, None, "invalid")
+ assert _helpers._get_uploaded_checksum_from_headers(response, None, None)
is None
+
+
def _mock_response(headers):
return mock.Mock(
headers=headers,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/google-resumable-media-2.5.0/tests/unit/test__upload.py
new/google-resumable-media-2.6.0/tests/unit/test__upload.py
--- old/google-resumable-media-2.5.0/tests/unit/test__upload.py 2023-04-24
21:00:19.000000000 +0200
+++ new/google-resumable-media-2.6.0/tests/unit/test__upload.py 2023-09-06
20:09:24.000000000 +0200
@@ -15,6 +15,7 @@
import http.client
import io
import sys
+import tempfile
from unittest import mock
import pytest # type: ignore
@@ -32,6 +33,26 @@
BASIC_CONTENT = "text/plain"
JSON_TYPE = "application/json; charset=UTF-8"
JSON_TYPE_LINE = b"content-type: application/json; charset=UTF-8\r\n"
+EXAMPLE_XML_UPLOAD_URL =
"https://test-project.storage.googleapis.com/test-bucket";
+EXAMPLE_HEADERS = {"example-key": "example-content"}
+EXAMPLE_XML_MPU_INITIATE_TEXT_TEMPLATE = """<?xml version="1.0"
encoding="UTF-8"?>
+<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/";>
+ <Bucket>travel-maps</Bucket>
+ <Key>paris.jpg</Key>
+ <UploadId>{upload_id}</UploadId>
+</InitiateMultipartUploadResult>
+"""
+UPLOAD_ID = "VXBsb2FkIElEIGZvciBlbHZpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"
+PARTS = {1: "39a59594290b0f9a30662a56d695b71d", 2:
"00000000290b0f9a30662a56d695b71d"}
+FILE_DATA = b"testdata" * 128
+
+
[email protected](scope="session")
+def filename():
+ with tempfile.NamedTemporaryFile() as f:
+ f.write(FILE_DATA)
+ f.flush()
+ yield f.name
class TestUploadBase(object):
@@ -1230,6 +1251,253 @@
assert result == "bytes 1000-10000/*"
+def test_xml_mpu_container_constructor_and_properties(filename):
+ container = _upload.XMLMPUContainer(EXAMPLE_XML_UPLOAD_URL, filename)
+ assert container.upload_url == EXAMPLE_XML_UPLOAD_URL
+ assert container.upload_id is None
+ assert container._headers == {}
+ assert container._parts == {}
+ assert container._filename == filename
+
+ container = _upload.XMLMPUContainer(
+ EXAMPLE_XML_UPLOAD_URL,
+ filename,
+ headers=EXAMPLE_HEADERS,
+ upload_id=UPLOAD_ID,
+ )
+ container._parts = PARTS
+ assert container.upload_url == EXAMPLE_XML_UPLOAD_URL
+ assert container.upload_id == UPLOAD_ID
+ assert container._headers == EXAMPLE_HEADERS
+ assert container._parts == PARTS
+ assert container._filename == filename
+
+
+def test_xml_mpu_container_initiate(filename):
+ container = _upload.XMLMPUContainer(
+ EXAMPLE_XML_UPLOAD_URL, filename, upload_id=UPLOAD_ID
+ )
+ with pytest.raises(ValueError):
+ container._prepare_initiate_request(BASIC_CONTENT)
+
+ container = _upload.XMLMPUContainer(
+ EXAMPLE_XML_UPLOAD_URL, filename, headers=EXAMPLE_HEADERS
+ )
+ verb, url, body, headers =
container._prepare_initiate_request(BASIC_CONTENT)
+ assert verb == _upload._POST
+ assert url == EXAMPLE_XML_UPLOAD_URL + _upload._MPU_INITIATE_QUERY
+ assert not body
+ assert headers == {**EXAMPLE_HEADERS, "content-type": BASIC_CONTENT}
+
+ _fix_up_virtual(container)
+ response = _make_xml_response(
+ text=EXAMPLE_XML_MPU_INITIATE_TEXT_TEMPLATE.format(upload_id=UPLOAD_ID)
+ )
+ container._process_initiate_response(response)
+ assert container.upload_id == UPLOAD_ID
+
+ with pytest.raises(NotImplementedError):
+ container.initiate(None, None)
+
+
+def test_xml_mpu_container_finalize(filename):
+ container = _upload.XMLMPUContainer(EXAMPLE_XML_UPLOAD_URL, filename)
+ with pytest.raises(ValueError):
+ container._prepare_finalize_request()
+
+ container = _upload.XMLMPUContainer(
+ EXAMPLE_XML_UPLOAD_URL,
+ filename,
+ headers=EXAMPLE_HEADERS,
+ upload_id=UPLOAD_ID,
+ )
+ container._parts = PARTS
+ verb, url, body, headers = container._prepare_finalize_request()
+ assert verb == _upload._POST
+ final_query = _upload._MPU_FINAL_QUERY_TEMPLATE.format(upload_id=UPLOAD_ID)
+ assert url == EXAMPLE_XML_UPLOAD_URL + final_query
+ assert headers == EXAMPLE_HEADERS
+ assert b"CompleteMultipartUpload" in body
+ for key, value in PARTS.items():
+ assert str(key).encode("utf-8") in body
+ assert value.encode("utf-8") in body
+
+ _fix_up_virtual(container)
+ response = _make_xml_response()
+ container._process_finalize_response(response)
+ assert container.finished
+
+ with pytest.raises(NotImplementedError):
+ container.finalize(None)
+
+
+def test_xml_mpu_container_cancel(filename):
+ container = _upload.XMLMPUContainer(EXAMPLE_XML_UPLOAD_URL, filename)
+ with pytest.raises(ValueError):
+ container._prepare_cancel_request()
+
+ container = _upload.XMLMPUContainer(
+ EXAMPLE_XML_UPLOAD_URL,
+ filename,
+ headers=EXAMPLE_HEADERS,
+ upload_id=UPLOAD_ID,
+ )
+ container._parts = PARTS
+ verb, url, body, headers = container._prepare_cancel_request()
+ assert verb == _upload._DELETE
+ final_query = _upload._MPU_FINAL_QUERY_TEMPLATE.format(upload_id=UPLOAD_ID)
+ assert url == EXAMPLE_XML_UPLOAD_URL + final_query
+ assert headers == EXAMPLE_HEADERS
+ assert not body
+
+ _fix_up_virtual(container)
+ response = _make_xml_response(status_code=204)
+ container._process_cancel_response(response)
+
+ with pytest.raises(NotImplementedError):
+ container.cancel(None)
+
+
+def test_xml_mpu_part(filename):
+ PART_NUMBER = 1
+ START = 0
+ END = 256
+ ETAG = PARTS[1]
+
+ part = _upload.XMLMPUPart(
+ EXAMPLE_XML_UPLOAD_URL,
+ UPLOAD_ID,
+ filename,
+ START,
+ END,
+ PART_NUMBER,
+ headers=EXAMPLE_HEADERS,
+ checksum="md5",
+ )
+ assert part.upload_url == EXAMPLE_XML_UPLOAD_URL
+ assert part.upload_id == UPLOAD_ID
+ assert part.filename == filename
+ assert part.etag is None
+ assert part.start == START
+ assert part.end == END
+ assert part.part_number == PART_NUMBER
+ assert part._headers == EXAMPLE_HEADERS
+ assert part._checksum_type == "md5"
+ assert part._checksum_object is None
+
+ part = _upload.XMLMPUPart(
+ EXAMPLE_XML_UPLOAD_URL,
+ UPLOAD_ID,
+ filename,
+ START,
+ END,
+ PART_NUMBER,
+ headers=EXAMPLE_HEADERS,
+ )
+ verb, url, payload, headers = part._prepare_upload_request()
+ assert verb == _upload._PUT
+ assert url == EXAMPLE_XML_UPLOAD_URL +
_upload._MPU_PART_QUERY_TEMPLATE.format(
+ part=PART_NUMBER, upload_id=UPLOAD_ID
+ )
+ assert headers == EXAMPLE_HEADERS
+ assert payload == FILE_DATA[START:END]
+
+ _fix_up_virtual(part)
+ response = _make_xml_response(headers={"etag": ETAG})
+ part._process_upload_response(response)
+ assert part.etag == ETAG
+
+
+def test_xml_mpu_part_invalid_response(filename):
+ PART_NUMBER = 1
+ START = 0
+ END = 256
+ ETAG = PARTS[1]
+
+ part = _upload.XMLMPUPart(
+ EXAMPLE_XML_UPLOAD_URL,
+ UPLOAD_ID,
+ filename,
+ START,
+ END,
+ PART_NUMBER,
+ headers=EXAMPLE_HEADERS,
+ checksum="md5",
+ )
+ _fix_up_virtual(part)
+ response = _make_xml_response(headers={"etag": ETAG})
+ with pytest.raises(common.InvalidResponse):
+ part._process_upload_response(response)
+
+
+def test_xml_mpu_part_checksum_failure(filename):
+ PART_NUMBER = 1
+ START = 0
+ END = 256
+ ETAG = PARTS[1]
+
+ part = _upload.XMLMPUPart(
+ EXAMPLE_XML_UPLOAD_URL,
+ UPLOAD_ID,
+ filename,
+ START,
+ END,
+ PART_NUMBER,
+ headers=EXAMPLE_HEADERS,
+ checksum="md5",
+ )
+ _fix_up_virtual(part)
+ part._prepare_upload_request()
+ response = _make_xml_response(
+ headers={"etag": ETAG, "x-goog-hash": "md5=Ojk9c3dhfxgoKVVHYwFbHQ=="}
+ ) # Example md5 checksum but not the correct one
+ with pytest.raises(common.DataCorruption):
+ part._process_upload_response(response)
+
+
+def test_xml_mpu_part_checksum_success(filename):
+ PART_NUMBER = 1
+ START = 0
+ END = 256
+ ETAG = PARTS[1]
+
+ part = _upload.XMLMPUPart(
+ EXAMPLE_XML_UPLOAD_URL,
+ UPLOAD_ID,
+ filename,
+ START,
+ END,
+ PART_NUMBER,
+ headers=EXAMPLE_HEADERS,
+ checksum="md5",
+ )
+ _fix_up_virtual(part)
+ part._prepare_upload_request()
+ response = _make_xml_response(
+ headers={"etag": ETAG, "x-goog-hash": "md5=pOUFGnohRRFFd24NztFuFw=="}
+ )
+ part._process_upload_response(response)
+ assert part.etag == ETAG
+ assert part.finished
+
+ # Test error handling
+ part = _upload.XMLMPUPart(
+ EXAMPLE_XML_UPLOAD_URL,
+ UPLOAD_ID,
+ filename,
+ START,
+ END,
+ PART_NUMBER,
+ headers=EXAMPLE_HEADERS,
+ checksum="md5",
+ )
+ with pytest.raises(NotImplementedError):
+ part.upload(None)
+ part._finished = True
+ with pytest.raises(ValueError):
+ part._prepare_upload_request()
+
+
def _make_response(status_code=http.client.OK, headers=None, metadata=None):
headers = headers or {}
return mock.Mock(
@@ -1239,6 +1507,16 @@
spec=["headers", "status_code"],
)
+
+def _make_xml_response(status_code=http.client.OK, headers=None, text=None):
+ headers = headers or {}
+ return mock.Mock(
+ headers=headers,
+ status_code=status_code,
+ text=text,
+ spec=["headers", "status_code"],
+ )
+
def _get_status_code(response):
return response.status_code
