Hello community, here is the log from the commit of package python-swiftclient for openSUSE:Factory checked in at 2019-12-04 13:49:02 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-swiftclient (Old) and /work/SRC/openSUSE:Factory/.python-swiftclient.new.4691 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-swiftclient" Wed Dec 4 13:49:02 2019 rev:27 rq:736670 version:3.8.1 Changes: -------- --- /work/SRC/openSUSE:Factory/python-swiftclient/python-swiftclient.changes 2019-05-03 22:43:21.907360024 +0200 +++ /work/SRC/openSUSE:Factory/.python-swiftclient.new.4691/python-swiftclient.changes 2019-12-04 14:19:01.262369397 +0100 @@ -1,0 +2,27 @@ +Wed Oct 9 13:07:20 UTC 2019 - cloud-de...@suse.de + +- update to version 3.8.1 + - Fix SLO re-upload + - Add Python 3 Train unit tests + - Clean up warnings from newer flake8 + - Optionally display listings in raw json + - docs: Fix warning treated as error + - Enable some off-by-default checks + - Add missing <sync-to> value in command line docs + - Delete/overwrite symlinks better + - Drag forward prettytable in lower-constraints + - Authors/changelog for 3.8.0 + - docs: Clean up formatting + - OpenDev Migration Patch + - Remove oslosphinx usage + - Authors/changelog for 3.8.1 + - Make proper functions instead of assigning lambdas + - Support pdb in tests better + - Replace git.openstack.org URLs with opendev.org URLs + - Update master for stable/stein + - Fix up requests so we can send non-RFC-compliant headers on py3 + - PDF Documentation Build tox target + - Isolate docs requirements + - Bump the openstackdocstheme extension to 1.20 + +------------------------------------------------------------------- Old: ---- python-swiftclient-3.7.0.tar.gz New: ---- python-swiftclient-3.8.1.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-swiftclient.spec ++++++ --- /var/tmp/diff_new_pack.RPe1v3/_old 2019-12-04 14:19:01.714369778 +0100 +++ /var/tmp/diff_new_pack.RPe1v3/_new 2019-12-04 14:19:01.718369781 +0100 @@ -17,14 +17,15 @@ Name: python-swiftclient -Version: 3.7.0 +Version: 3.8.1 Release: 0 Summary: OpenStack Object Storage API Client Library License: Apache-2.0 Group: Development/Languages/Python URL: https://launchpad.net/python-swiftclient -Source0: https://files.pythonhosted.org/packages/source/p/python-swiftclient/python-swiftclient-3.7.0.tar.gz +Source0: https://files.pythonhosted.org/packages/source/p/python-swiftclient/python-swiftclient-3.8.1.tar.gz BuildRequires: openstack-macros +BuildRequires: python2-futures >= 3.0.0 BuildRequires: python2-keystoneclient BuildRequires: python2-mock BuildRequires: python2-pbr @@ -60,10 +61,8 @@ %package -n python-swiftclient-doc Summary: %{summary} - Documentation Group: Documentation/HTML -BuildRequires: python-Sphinx -BuildRequires: python-futures >= 3.0.0 -BuildRequires: python-openstackdocstheme -Requires: %{name} = %{version} +BuildRequires: python3-Sphinx +BuildRequires: python3-openstackdocstheme %description -n python-swiftclient-doc This is a python client for the Swift API. There's a Python API (the @@ -72,12 +71,12 @@ This package contains documentation files for %{name}. %prep -%autosetup -p1 -n python-swiftclient-3.7.0 +%autosetup -p1 -n python-swiftclient-3.8.1 %py_req_cleanup %build %{python_build} -%{__python2} setup.py build_sphinx +PBR_VERSION=%{version} %sphinx_build -b html doc/source doc/build/html rm -rf doc/build/html/.{doctrees,buildinfo} %install ++++++ _service ++++++ --- /var/tmp/diff_new_pack.RPe1v3/_old 2019-12-04 14:19:01.742369801 +0100 +++ /var/tmp/diff_new_pack.RPe1v3/_new 2019-12-04 14:19:01.742369801 +0100 @@ -1,8 +1,8 @@ <services> <service mode="disabled" name="renderspec"> - <param name="input-template">https://raw.githubusercontent.com/openstack/rpm-packaging/stable/stein/openstack/python-swiftclient/python-swiftclient.spec.j2</param> + <param name="input-template">https://raw.githubusercontent.com/openstack/rpm-packaging/stable/train/openstack/python-swiftclient/python-swiftclient.spec.j2</param> <param name="output-name">python-swiftclient.spec</param> - <param name="requirements">https://raw.githubusercontent.com/openstack/python-swiftclient/stable/stein/requirements.txt</param> + <param name="requirements">https://raw.githubusercontent.com/openstack/python-swiftclient/stable/train/requirements.txt</param> <param name="changelog-email">cloud-de...@suse.de</param> <param name="changelog-provider">gh,openstack,python-swiftclient</param> </service> ++++++ python-swiftclient-3.7.0.tar.gz -> python-swiftclient-3.8.1.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/.zuul.yaml new/python-swiftclient-3.8.1/.zuul.yaml --- old/python-swiftclient-3.7.0/.zuul.yaml 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/.zuul.yaml 2019-09-13 23:39:55.000000000 +0200 @@ -9,7 +9,7 @@ # job sets zuul_work_dir to the swift directory and uses tox # for installation. required-projects: - - git.openstack.org/openstack/python-swiftclient + - opendev.org/openstack/python-swiftclient - job: name: swiftclient-functional @@ -19,10 +19,10 @@ python-swiftclient installed from source instead as package from PyPI. required-projects: - - git.openstack.org/openstack/python-swiftclient + - opendev.org/openstack/python-swiftclient vars: # Override value from parent job to use swiftclient tests - zuul_work_dir: "{{ zuul.projects['git.openstack.org/openstack/python-swiftclient'].src_dir }}" + zuul_work_dir: "{{ zuul.projects['opendev.org/openstack/python-swiftclient'].src_dir }}" - job: name: swiftclient-functional-py2 @@ -39,9 +39,7 @@ - openstack-lower-constraints-jobs - openstack-pypy-jobs-nonvoting - openstack-python-jobs - - openstack-python35-jobs - - openstack-python36-jobs - - openstack-python37-jobs + - openstack-python3-train-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/AUTHORS new/python-swiftclient-3.8.1/AUTHORS --- old/python-swiftclient-3.7.0/AUTHORS 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/AUTHORS 2019-09-13 23:39:58.000000000 +0200 @@ -24,6 +24,7 @@ Claudiu Belu (cb...@cloudbasesolutions.com) Clay Gerrard (clay.gerr...@gmail.com) Clint Byrum (cl...@fewbar.com) +Corey Bryant (corey.bry...@canonical.com) Dan Prince (dpri...@redhat.com) Daniel Wakefield (daniel.wakefi...@hp.com) Darrell Bishop (darr...@swiftstack.com) @@ -52,6 +53,7 @@ howardlee (lihongwe...@inspur.com) Hu Bing (hubin...@cn.ibm.com) Ian Cordasco (ian.corda...@rackspace.com) +jacky06 (zhang....@99cloud.net) Jaivish Kothari (jaivish.koth...@nectechnologies.in) Jakub Krajcovic (jakub.krajco...@gmail.com) James Nzomo (james@tdt.rocks) @@ -99,6 +101,7 @@ Pallavi (pallav...@nectechnologies.in) Paul Belanger (pabelan...@redhat.com) Paulo Ewerton (pauloewer...@lsd.ufcg.edu.br) +pengyuesheng (pengyuesh...@gohighsec.com) Pete Zaitcev (zait...@kotori.zaitcev.us) Peter Lisak (peter.li...@firma.seznam.cz) Petr Kovar (pko...@redhat.com) @@ -147,6 +150,7 @@ Vu Cong Tuan (tua...@vn.fujitsu.com) wangqi (wang...@99cloud.net) wangxiyuan (wangxiy...@huawei.com) +wangzhenyu (wan...@fiberhome.com) Wu Wenxiang (wu.wenxi...@99cloud.net) wu.chunyang (wu.chuny...@99cloud.net) YangLei (yang...@cn.ibm.com) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/ChangeLog new/python-swiftclient-3.8.1/ChangeLog --- old/python-swiftclient-3.7.0/ChangeLog 2019-03-05 14:10:22.000000000 +0100 +++ new/python-swiftclient-3.8.1/ChangeLog 2019-09-13 23:39:58.000000000 +0200 @@ -1,3 +1,30 @@ +3.8.1 +----- + +* Deleting or overwriting a symlink to an SLO or DLO will no longer attempt + to clean up the large object's segments. + +* Fixed an issue sending non-ASCII metadata keys on Python 3. + Note that receiving such metadata on py3 is still broken; + see https://bugs.python.org/issue37093 + +* Documentation can now be rendered as a PDF. + +* Dropped Python 3.5 testing. + + +3.8.0 +----- + +* Added a new `--json` option to `swift list`. + +* Fixed an issue introduced in 3.5.0 where re-uploading an SLO with + the same size, mtime, and segment size would delete all of the + just-uploaded segments. + +* Various other minor bug fixes and improvements. + + 3.7.0 ----- diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/PKG-INFO new/python-swiftclient-3.8.1/PKG-INFO --- old/python-swiftclient-3.7.0/PKG-INFO 2019-03-05 14:11:12.000000000 +0100 +++ new/python-swiftclient-3.8.1/PKG-INFO 2019-09-13 23:40:49.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: python-swiftclient -Version: 3.7.0 +Version: 3.8.1 Summary: OpenStack Object Storage API Client Library Home-page: https://docs.openstack.org/python-swiftclient/latest/ Author: OpenStack @@ -52,7 +52,7 @@ .. _Launchpad project: https://launchpad.net/python-swiftclient .. _Blueprints: https://blueprints.launchpad.net/python-swiftclient .. _Bugs: https://bugs.launchpad.net/python-swiftclient - .. _Source: https://git.openstack.org/cgit/openstack/python-swiftclient + .. _Source: https://opendev.org/openstack/python-swiftclient .. _How to Contribute: https://docs.openstack.org/infra/manual/developers.html .. _Specs: https://specs.openstack.org/openstack/swift-specs/ .. _Release Notes: https://docs.openstack.org/releasenotes/python-swiftclient @@ -72,7 +72,6 @@ Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: 3.7 Provides-Extra: test diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/README.rst new/python-swiftclient-3.8.1/README.rst --- old/python-swiftclient-3.7.0/README.rst 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/README.rst 2019-09-13 23:39:55.000000000 +0200 @@ -44,7 +44,7 @@ .. _Launchpad project: https://launchpad.net/python-swiftclient .. _Blueprints: https://blueprints.launchpad.net/python-swiftclient .. _Bugs: https://bugs.launchpad.net/python-swiftclient -.. _Source: https://git.openstack.org/cgit/openstack/python-swiftclient +.. _Source: https://opendev.org/openstack/python-swiftclient .. _How to Contribute: https://docs.openstack.org/infra/manual/developers.html .. _Specs: https://specs.openstack.org/openstack/swift-specs/ .. _Release Notes: https://docs.openstack.org/releasenotes/python-swiftclient diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/doc/requirements.txt new/python-swiftclient-3.8.1/doc/requirements.txt --- old/python-swiftclient-3.7.0/doc/requirements.txt 1970-01-01 01:00:00.000000000 +0100 +++ new/python-swiftclient-3.8.1/doc/requirements.txt 2019-09-13 23:39:55.000000000 +0200 @@ -0,0 +1,5 @@ +keystoneauth1>=3.4.0 # Apache-2.0 +sphinx!=1.6.6,!=1.6.7,<2.0.0,>=1.6.2;python_version=='2.7' # BSD +sphinx!=1.6.6,!=1.6.7,!=2.1.0,>=1.6.2;python_version>='3.4' # BSD +reno>=2.5.0 # Apache-2.0 +openstackdocstheme>=1.20.0 # Apache-2.0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/doc/source/cli/index.rst new/python-swiftclient-3.8.1/doc/source/cli/index.rst --- old/python-swiftclient-3.7.0/doc/source/cli/index.rst 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/doc/source/cli/index.rst 2019-09-13 23:39:55.000000000 +0200 @@ -245,13 +245,10 @@ --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \ list -.. We need the backslash below in order to indent the note -\ +.. note:: - .. note:: - - Leftover environment variables are a common source of confusion when - authorization fails. + Leftover environment variables are a common source of confusion when + authorization fails. CLI commands ~~~~~~~~~~~~ @@ -446,7 +443,7 @@ .. code-block:: console - Usage: swift post [--read-acl <acl>] [--write-acl <acl>] [--sync-to] + Usage: swift post [--read-acl <acl>] [--write-acl <acl>] [--sync-to <sync-to>] [--sync-key <sync-key>] [--meta <name:value>] [--header <header>] [<container> [<object>]] @@ -739,15 +736,15 @@ But beyond that, ``time`` can also be specified as an ISO 8601 timestamp in one of following formats: - i) Complete date: YYYY-MM-DD (eg 1997-07-16) +i) Complete date: YYYY-MM-DD (e.g. 1997-07-16) - ii) Complete date plus hours, minutes and seconds: - YYYY-MM-DDThh:mm:ss - (eg 1997-07-16T19:20:30) - - iii) Complete date plus hours, minutes and seconds with UTC designator: - YYYY-MM-DDThh:mm:ssZ - (eg 1997-07-16T19:20:30Z) +ii) Complete date plus hours, minutes and seconds: + YYYY-MM-DDThh:mm:ss + (e.g. 1997-07-16T19:20:30) + +iii) Complete date plus hours, minutes and seconds with UTC designator: + YYYY-MM-DDThh:mm:ssZ + (e.g. 1997-07-16T19:20:30Z) Please be aware that if you don't provide the UTC designator (i.e., Z) the timestamp is generated using your local timezone. If only a date is @@ -881,17 +878,14 @@ testSwift.txt [auth 0.028s, headers 0.045s, total 0.045s, 0.002 MB/s] -.. We need the backslash below in order to indent the note -\ - - .. note:: +.. note:: - To upload an object to a container, your current working directory must be - where the file is located or you must provide the complete path to the file. - In other words, the --object-name <object-name> is an option that will upload - file and name object to <object-name> or upload directory and use <object-name> as - object prefix. In the case that you provide the complete path of the file, - that complete path will be the name of the uploaded object. + To upload an object to a container, your current working directory must be + where the file is located or you must provide the complete path to the file. + In other words, the --object-name <object-name> is an option that will upload + file and name object to <object-name> or upload directory and use <object-name> as + object prefix. In the case that you provide the complete path of the file, + that complete path will be the name of the uploaded object. For example: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/doc/source/conf.py new/python-swiftclient-3.8.1/doc/source/conf.py --- old/python-swiftclient-3.7.0/doc/source/conf.py 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/doc/source/conf.py 2019-09-13 23:39:55.000000000 +0200 @@ -53,17 +53,8 @@ master_doc = 'index' # General information about the project. -project = u'Swiftclient' copyright = u'2013-2016 OpenStack, LLC.' -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -import swiftclient.version -release = swiftclient.version.version_string -version = swiftclient.version.version_string - # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None @@ -190,7 +181,7 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]) latex_documents = [ - ('index', 'SwiftClient.tex', u'SwiftClient Documentation', + ('index', 'doc-python-swiftclient.tex', u'SwiftClient Documentation', u'OpenStack, LLC.', 'manual'), ] @@ -210,3 +201,5 @@ # If false, no module index is generated. # latex_use_modindex = True + +latex_use_xindy = False diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/doc/source/index.rst new/python-swiftclient-3.8.1/doc/source/index.rst --- old/python-swiftclient-3.7.0/doc/source/index.rst 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/doc/source/index.rst 2019-09-13 23:39:55.000000000 +0200 @@ -39,17 +39,17 @@ License ~~~~~~~ - Copyright 2013 OpenStack, LLC. +Copyright 2013 OpenStack, LLC. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +* http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/doc/source/introduction.rst new/python-swiftclient-3.8.1/doc/source/introduction.rst --- old/python-swiftclient-3.7.0/doc/source/introduction.rst 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/doc/source/introduction.rst 2019-09-13 23:39:55.000000000 +0200 @@ -16,41 +16,41 @@ Alongside the command line tool, the ``python-swiftclient`` includes two levels of API: - * A low level client API that provides simple Python wrappers around the - various authentication mechanisms and the individual HTTP requests. - * A high level service API that provides methods for performing common - operations in parallel on a thread pool. +* A low level client API that provides simple Python wrappers around the + various authentication mechanisms and the individual HTTP requests. +* A high level service API that provides methods for performing common + operations in parallel on a thread pool. Example use cases: - * Uploading and retrieving data - Use the command line tool if you are simply uploading and downloading - files and directories to and from your filesystem. The command line tool - can be integrated into a shell script to automate tasks. - - * Integrating into an automated Python workflow - Use the ``SwiftService`` API to perform operations offered by the CLI - if your use case requires integration with a Python-based workflow. - This method offers greater control and flexibility over individual object - operations, such as the metadata set on each object. The ``SwiftService`` - class provides methods to perform multiple sets of operations against a - swift object store using a configurable shared thread pool. A single - instance of the ``SwiftService`` class can be shared between multiple - threads in your own code. - - * Developing an application in Python to access a swift object store - Use the ``SwiftService`` API to develop Python applications that use - swift to store and retrieve objects. A ``SwiftService`` instance provides - a configurable thread pool for performing all operations supported by the - CLI. - - * Fine-grained control over threading or the requests being performed - Use the ``Connection`` API if your use case requires fine grained control - over advanced features or you wish to use your own existing threading - model. Examples of advanced features requiring the use of the - ``Connection`` API include creating an SLO manifest that references - already existing objects, or fine grained control over the query strings - supplied with each HTTP request. +* Uploading and retrieving data + Use the command line tool if you are simply uploading and downloading + files and directories to and from your filesystem. The command line tool + can be integrated into a shell script to automate tasks. + +* Integrating into an automated Python workflow + Use the ``SwiftService`` API to perform operations offered by the CLI + if your use case requires integration with a Python-based workflow. + This method offers greater control and flexibility over individual object + operations, such as the metadata set on each object. The ``SwiftService`` + class provides methods to perform multiple sets of operations against a + swift object store using a configurable shared thread pool. A single + instance of the ``SwiftService`` class can be shared between multiple + threads in your own code. + +* Developing an application in Python to access a swift object store + Use the ``SwiftService`` API to develop Python applications that use + swift to store and retrieve objects. A ``SwiftService`` instance provides + a configurable thread pool for performing all operations supported by the + CLI. + +* Fine-grained control over threading or the requests being performed + Use the ``Connection`` API if your use case requires fine grained control + over advanced features or you wish to use your own existing threading + model. Examples of advanced features requiring the use of the + ``Connection`` API include creating an SLO manifest that references + already existing objects, or fine grained control over the query strings + supplied with each HTTP request. Important considerations ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -66,19 +66,19 @@ There are two main restrictions to bear in mind when designing an application that uses an object store: - * You cannot rename objects. Due to fact that the name of an object is one - of the factors that determines where the object and its replicas are stored, - renaming would require multiple copies of the data to be moved between - physical storage devices. If you want to rename an object you must upload - to the new location, or make a server side copy request to the new location, - and then delete the original. - - * You cannot modify objects. Objects are stored in multiple locations and - are checked for integrity based on the MD5 sum calculated during - upload. In order to modify the contents of an object, the entire desired - contents must be re-uploaded. In certain special cases it is possible to - work around this restriction using large objects, but no general - file-like access is available to modify a stored object. +* You cannot rename objects. Due to fact that the name of an object is one + of the factors that determines where the object and its replicas are stored, + renaming would require multiple copies of the data to be moved between + physical storage devices. If you want to rename an object you must upload + to the new location, or make a server side copy request to the new location, + and then delete the original. + +* You cannot modify objects. Objects are stored in multiple locations and + are checked for integrity based on the MD5 sum calculated during + upload. In order to modify the contents of an object, the entire desired + contents must be re-uploaded. In certain special cases it is possible to + work around this restriction using large objects, but no general + file-like access is available to modify a stored object. Objects cannot be locked ------------------------ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/doc/source/service-api.rst new/python-swiftclient-3.8.1/doc/source/service-api.rst --- old/python-swiftclient-3.7.0/doc/source/service-api.rst 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/doc/source/service-api.rst 2019-09-13 23:39:55.000000000 +0200 @@ -26,10 +26,10 @@ supplying options from multiple different auth versions can cause unexpected behaviour. - .. note:: +.. note:: - Leftover environment variables are a common source of confusion when - authorization fails. + Leftover environment variables are a common source of confusion when + authorization fails. Keystone V3 ~~~~~~~~~~~ @@ -109,17 +109,17 @@ Options ~~~~~~~ - ``retries``: ``5`` +``retries``: ``5`` The number of times that the library should attempt to retry HTTP actions before giving up and reporting a failure. - ``container_threads``: ``10`` +``container_threads``: ``10`` - ``object_dd_threads``: ``10`` +``object_dd_threads``: ``10`` - ``object_uu_threads``: ``10`` +``object_uu_threads``: ``10`` - ``segment_threads``: ``10`` +``segment_threads``: ``10`` The above options determine the size of the available thread pools for performing swift operations. Container operations (such as listing a container) operate in the container threads, and a similar pattern @@ -131,86 +131,86 @@ ``uu`` and ``dd``. This stands for "upload/update" and "download/delete", and the corresponding actions will be run on separate threads pools. - ``segment_size``: ``None`` +``segment_size``: ``None`` If specified, this option enables uploading of large objects. Should the object being uploaded be larger than 5G in size, this option is mandatory otherwise the upload will fail. This option should be specified as a size in bytes. - ``use_slo``: ``False`` +``use_slo``: ``False`` Used in combination with the above option, ``use_slo`` will upload large objects as static rather than dynamic. Only static large objects provide error checking for the downloaded object, so we recommend this option. - ``segment_container``: ``None`` +``segment_container``: ``None`` Allows the user to select the container into which large object segments will be uploaded. We do not recommend changing this value as it could make locating orphaned segments more difficult in the case of errors. - ``leave_segments``: ``False`` +``leave_segments``: ``False`` Setting this option to true means that when deleting or overwriting a large object, its segments will be left in the object store and must be cleaned up manually. This option can be useful when sharing large object segments between multiple objects in more advanced scenarios, but must be treated with care, as it could lead to ever increasing storage usage. - ``changed``: ``None`` +``changed``: ``None`` This option affects uploads and simply means that those objects which already exist in the object store will not be overwritten if the ``mtime`` and size of the source is the same as the existing object. - ``skip_identical``: ``False`` +``skip_identical``: ``False`` A slightly more thorough case of the above, but rather than ``mtime`` and size uses an object's ``MD5 sum``. - ``yes_all``: ``False`` +``yes_all``: ``False`` This options affects only download and delete, and in each case must be specified in order to download/delete the entire contents of an account. This option has no effect on any other calls. - ``no_download``: ``False`` +``no_download``: ``False`` This option only affects download and means that all operations proceed as normal with the exception that no data is written to disk. - ``header``: ``[]`` +``header``: ``[]`` Used with upload and post operations to set headers on objects. Headers are specified as colon separated strings, e.g. "content-type:text/plain". - ``meta``: ``[]`` +``meta``: ``[]`` Used to set metadata on an object similarly to headers. .. note:: Setting metadata is a destructive operation, so when updating one of many metadata values all desired metadata for an object must be re-applied. - ``long``: ``False`` +``long``: ``False`` Affects only list operations, and results in more metrics being made available in the results at the expense of lower performance. - ``fail_fast``: ``False`` +``fail_fast``: ``False`` Applies to delete and upload operations, and attempts to abort queued tasks in the event of errors. - ``prefix``: ``None`` +``prefix``: ``None`` Affects list operations; only objects with the given prefix will be returned/affected. It is not advisable to set at the service level, as those operations that call list to discover objects on which they should operate will also be affected. - ``delimiter``: ``None`` +``delimiter``: ``None`` Affects list operations, and means that listings only contain results up to the first instance of the delimiter in the object name. This is useful for working with objects containing '/' in their names to simulate folder structures. - ``dir_marker``: ``False`` +``dir_marker``: ``False`` Affects uploads, and allows empty 'pseudofolder' objects to be created when the source of an upload is ``None``. - ``checksum``: ``True`` +``checksum``: ``True`` Affects uploads and downloads. If set check md5 sum for the transfer. - ``shuffle``: ``False`` +``shuffle``: ``False`` When downloading objects, the default behaviour of the CLI is to shuffle lists of objects in order to spread the load on storage drives when multiple clients are downloading the same files to multiple locations (e.g. in the @@ -220,12 +220,12 @@ are downloaded in lexically-sorted order. Setting this option to ``True`` gives the same shuffling behaviour as the CLI. - ``destination``: ``None`` +``destination``: ``None`` When copying objects, this specifies the destination where the object will be copied to. The default of None means copy will be the same as source. - ``fresh_metadata``: ``None`` +``fresh_metadata``: ``None`` When copying objects, this specifies that the object metadata on the source will *not* be applied to the destination object - the destination object will have a new fresh set of metadata that includes diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/lower-constraints.txt new/python-swiftclient-3.8.1/lower-constraints.txt --- old/python-swiftclient-3.7.0/lower-constraints.txt 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/lower-constraints.txt 2019-09-13 23:39:55.000000000 +0200 @@ -20,12 +20,11 @@ mccabe==0.2.1 mock==1.2.0 netaddr==0.7.10 -openstackdocstheme==1.18.1 +openstackdocstheme==1.20.0 oslo.config==1.2.0 -oslosphinx==4.7.0 pbr==2.0.0 pep8==1.5.7 -PrettyTable==0.7 +PrettyTable==0.7.1 pyflakes==0.8.1 Pygments==2.2.0 python-keystoneclient==0.7.0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/python_swiftclient.egg-info/PKG-INFO new/python-swiftclient-3.8.1/python_swiftclient.egg-info/PKG-INFO --- old/python-swiftclient-3.7.0/python_swiftclient.egg-info/PKG-INFO 2019-03-05 14:11:12.000000000 +0100 +++ new/python-swiftclient-3.8.1/python_swiftclient.egg-info/PKG-INFO 2019-09-13 23:40:49.000000000 +0200 @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: python-swiftclient -Version: 3.7.0 +Version: 3.8.1 Summary: OpenStack Object Storage API Client Library Home-page: https://docs.openstack.org/python-swiftclient/latest/ Author: OpenStack @@ -52,7 +52,7 @@ .. _Launchpad project: https://launchpad.net/python-swiftclient .. _Blueprints: https://blueprints.launchpad.net/python-swiftclient .. _Bugs: https://bugs.launchpad.net/python-swiftclient - .. _Source: https://git.openstack.org/cgit/openstack/python-swiftclient + .. _Source: https://opendev.org/openstack/python-swiftclient .. _How to Contribute: https://docs.openstack.org/infra/manual/developers.html .. _Specs: https://specs.openstack.org/openstack/swift-specs/ .. _Release Notes: https://docs.openstack.org/releasenotes/python-swiftclient @@ -72,7 +72,6 @@ Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: 3.7 Provides-Extra: test diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/python_swiftclient.egg-info/SOURCES.txt new/python-swiftclient-3.8.1/python_swiftclient.egg-info/SOURCES.txt --- old/python-swiftclient-3.7.0/python_swiftclient.egg-info/SOURCES.txt 2019-03-05 14:11:12.000000000 +0100 +++ new/python-swiftclient-3.8.1/python_swiftclient.egg-info/SOURCES.txt 2019-09-13 23:40:49.000000000 +0200 @@ -21,6 +21,7 @@ tox.ini bin/swift doc/Makefile +doc/requirements.txt doc/manpages/swift.1 doc/source/client-api.rst doc/source/conf.py @@ -28,6 +29,7 @@ doc/source/introduction.rst doc/source/service-api.rst doc/source/swiftclient.rst +doc/source/_static/.gitignore doc/source/_templates/.empty doc/source/cli/index.rst examples/capabilities.py @@ -52,6 +54,8 @@ releasenotes/notes/350_notes-ad0ae19704b2eb88.yaml releasenotes/notes/360_notes-1ec385df13a3a735.yaml releasenotes/notes/361_notes-59e020e68bcdd709.yaml +releasenotes/notes/3_8_0_release-bd867fbdb8c895d3.yaml +releasenotes/notes/3_8_1_release-cb5648c3ae69bde1.yaml releasenotes/source/conf.py releasenotes/source/current.rst releasenotes/source/index.rst @@ -60,6 +64,7 @@ releasenotes/source/pike.rst releasenotes/source/queens.rst releasenotes/source/rocky.rst +releasenotes/source/stein.rst swiftclient/__init__.py swiftclient/authv1.py swiftclient/client.py diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/python_swiftclient.egg-info/pbr.json new/python-swiftclient-3.8.1/python_swiftclient.egg-info/pbr.json --- old/python-swiftclient-3.7.0/python_swiftclient.egg-info/pbr.json 2019-03-05 14:11:12.000000000 +0100 +++ new/python-swiftclient-3.8.1/python_swiftclient.egg-info/pbr.json 2019-09-13 23:40:49.000000000 +0200 @@ -1 +1 @@ -{"git_version": "991a6ce", "is_release": true} \ No newline at end of file +{"git_version": "72b90fe", "is_release": true} \ No newline at end of file diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/python_swiftclient.egg-info/requires.txt new/python-swiftclient-3.8.1/python_swiftclient.egg-info/requires.txt --- old/python-swiftclient-3.7.0/python_swiftclient.egg-info/requires.txt 2019-03-05 14:11:12.000000000 +0100 +++ new/python-swiftclient-3.8.1/python_swiftclient.egg-info/requires.txt 2019-09-13 23:40:49.000000000 +0200 @@ -12,8 +12,4 @@ coverage!=4.4,>=4.0 keystoneauth1>=3.4.0 mock>=1.2.0 -oslosphinx>=4.7.0 -sphinx!=1.6.6,!=1.6.7,>=1.6.2 stestr>=2.0.0 -reno>=2.5.0 -openstackdocstheme>=1.18.1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/releasenotes/notes/3_8_0_release-bd867fbdb8c895d3.yaml new/python-swiftclient-3.8.1/releasenotes/notes/3_8_0_release-bd867fbdb8c895d3.yaml --- old/python-swiftclient-3.7.0/releasenotes/notes/3_8_0_release-bd867fbdb8c895d3.yaml 1970-01-01 01:00:00.000000000 +0100 +++ new/python-swiftclient-3.8.1/releasenotes/notes/3_8_0_release-bd867fbdb8c895d3.yaml 2019-09-13 23:39:55.000000000 +0200 @@ -0,0 +1,9 @@ +--- +features: + - | + Added a new ``--json`` option to ``swift list``. +fixes: + - | + Fixed an issue introduced in 3.5.0 where re-uploading an SLO with + the same size, mtime, and segment size would delete all of the + just-uploaded segments. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/releasenotes/notes/3_8_1_release-cb5648c3ae69bde1.yaml new/python-swiftclient-3.8.1/releasenotes/notes/3_8_1_release-cb5648c3ae69bde1.yaml --- old/python-swiftclient-3.7.0/releasenotes/notes/3_8_1_release-cb5648c3ae69bde1.yaml 1970-01-01 01:00:00.000000000 +0100 +++ new/python-swiftclient-3.8.1/releasenotes/notes/3_8_1_release-cb5648c3ae69bde1.yaml 2019-09-13 23:39:58.000000000 +0200 @@ -0,0 +1,14 @@ +--- +fixes: + - | + Deleting or overwriting a symlink to an SLO or DLO will no longer attempt + to clean up the large object's segments. + - | + Fixed an issue sending non-ASCII metadata keys on Python 3. + Note that *receiving* such metadata on py3 is `still broken + <https://bugs.python.org/issue37093>`__. +other: + - | + Documentation can now be rendered as a PDF. + - | + Dropped Python 3.5 testing. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/releasenotes/source/conf.py new/python-swiftclient-3.8.1/releasenotes/source/conf.py --- old/python-swiftclient-3.7.0/releasenotes/source/conf.py 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/releasenotes/source/conf.py 2019-09-13 23:39:55.000000000 +0200 @@ -65,15 +65,8 @@ master_doc = 'index' # General information about the project. -project = u'Swift Client Release Notes' copyright = u'%d, OpenStack Foundation' % datetime.datetime.now().year -# Release notes are version independent. -# The short X.Y version. -version = '' -# The full version, including alpha/beta/rc tags. -release = '' - # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # @@ -173,11 +166,6 @@ # # html_extra_path = [] -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' -html_last_updated_fmt = '%Y-%m-%d %H:%M' - # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/releasenotes/source/index.rst new/python-swiftclient-3.8.1/releasenotes/source/index.rst --- old/python-swiftclient-3.7.0/releasenotes/source/index.rst 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/releasenotes/source/index.rst 2019-09-13 23:39:55.000000000 +0200 @@ -6,6 +6,7 @@ :maxdepth: 1 current + stein rocky queens pike diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/releasenotes/source/stein.rst new/python-swiftclient-3.8.1/releasenotes/source/stein.rst --- old/python-swiftclient-3.7.0/releasenotes/source/stein.rst 1970-01-01 01:00:00.000000000 +0100 +++ new/python-swiftclient-3.8.1/releasenotes/source/stein.rst 2019-09-13 23:39:55.000000000 +0200 @@ -0,0 +1,6 @@ +=================================== + Stein Series Release Notes +=================================== + +.. release-notes:: + :branch: stable/stein diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/setup.cfg new/python-swiftclient-3.8.1/setup.cfg --- old/python-swiftclient-3.7.0/setup.cfg 2019-03-05 14:11:12.000000000 +0100 +++ new/python-swiftclient-3.8.1/setup.cfg 2019-09-13 23:40:49.000000000 +0200 @@ -17,7 +17,6 @@ Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/swiftclient/client.py new/python-swiftclient-3.8.1/swiftclient/client.py --- old/python-swiftclient-3.7.0/swiftclient/client.py 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/swiftclient/client.py 2019-09-13 23:39:55.000000000 +0200 @@ -39,7 +39,7 @@ # Default is 100, increase to 256 http_client._MAXHEADERS = 256 -VERSIONFUL_AUTH_PATH = re.compile('v[2-3](?:\.0)?$') +VERSIONFUL_AUTH_PATH = re.compile(r'v[2-3](?:\.0)?$') AUTH_VERSIONS_V1 = ('1.0', '1', 1) AUTH_VERSIONS_V2 = ('2.0', '2', 2) AUTH_VERSIONS_V3 = ('3.0', '3', 3) @@ -74,8 +74,10 @@ pass # requests version 1.2.3 try to encode headers in ascii, preventing -# utf-8 encoded header to be 'prepared' -if StrictVersion(requests.__version__) < StrictVersion('2.0.0'): +# utf-8 encoded header to be 'prepared'. This also affects all +# (or at least most) versions of requests on py3 +if StrictVersion(requests.__version__) < StrictVersion('2.0.0') \ + or not six.PY2: from requests.structures import CaseInsensitiveDict def prepare_unicode_headers(self, headers): @@ -1885,7 +1887,9 @@ reset = getattr(contents, 'reset', None) if tell and seek: orig_pos = tell() - reset_func = lambda *a, **k: seek(orig_pos) + + def reset_func(*a, **kw): + seek(orig_pos) elif reset: reset_func = reset return self._retry(reset_func, put_object, container, obj, contents, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/swiftclient/multithreading.py new/python-swiftclient-3.8.1/swiftclient/multithreading.py --- old/python-swiftclient-3.7.0/swiftclient/multithreading.py 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/swiftclient/multithreading.py 2019-09-13 23:39:55.000000000 +0200 @@ -168,6 +168,12 @@ We will only create as many connections as are required concurrently. """ def __init__(self, create_connection, max_workers): + """ + Initializes a new ThreadPoolExecutor instance. + + :param create_connection: callable to use to create new connections + :param max_workers: the maximum number of threads that can be used + """ self._connections = PriorityQueue() self._create_connection = create_connection for p in range(0, max_workers): @@ -175,6 +181,14 @@ super(ConnectionThreadPoolExecutor, self).__init__(max_workers) def submit(self, fn, *args, **kwargs): + """ + Schedules the callable, `fn`, to be executed + + :param fn: the callable to be invoked + :param args: the positional arguments for the callable + :param kwargs: the keyword arguments for the callable + :returns: a Future object representing the execution of the callable + """ def conn_fn(): priority = None conn = None diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/swiftclient/service.py new/python-swiftclient-3.8.1/swiftclient/service.py --- old/python-swiftclient-3.7.0/swiftclient/service.py 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/swiftclient/service.py 2019-09-13 23:39:55.000000000 +0200 @@ -17,6 +17,7 @@ import os +from collections import defaultdict from concurrent.futures import as_completed, CancelledError, TimeoutError from copy import deepcopy from errno import EEXIST, ENOENT @@ -44,7 +45,7 @@ from swiftclient.utils import ( config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG, parse_api_response, report_traceback, n_groups, split_request_headers, - n_at_a_time + n_at_a_time, normalize_manifest_path ) from swiftclient.exceptions import ClientException from swiftclient.multithreading import MultiThreadingManager @@ -173,6 +174,7 @@ 'container_threads': 10 } + _default_global_options = _build_default_global_options() _default_local_options = { @@ -451,7 +453,9 @@ **_default_local_options ) process_options(self._options) - create_connection = lambda: get_conn(self._options) + + def create_connection(): + return get_conn(self._options) self.thread_manager = MultiThreadingManager( create_connection, segment_threads=self._options['segment_threads'], @@ -2066,14 +2070,13 @@ 'status': 'skipped-changed' }) return res - if not options['leave_segments']: + if not options['leave_segments'] and not headers.get( + 'content-location'): old_manifest = headers.get('x-object-manifest') if is_slo: - for old_seg in chunk_data: - seg_path = old_seg['name'].lstrip('/') - if isinstance(seg_path, text_type): - seg_path = seg_path.encode('utf-8') - old_slo_manifest_paths.append(seg_path) + old_slo_manifest_paths.extend( + normalize_manifest_path(old_seg['name']) + for old_seg in chunk_data) except ClientException as err: if err.http_status != 404: traceback, err_time = report_traceback() @@ -2163,8 +2166,9 @@ response = self._upload_slo_manifest( conn, segment_results, container, obj, put_headers) res['manifest_response_dict'] = response - new_slo_manifest_paths = { - seg['segment_location'] for seg in segment_results} + new_slo_manifest_paths.update( + normalize_manifest_path(new_seg['segment_location']) + for new_seg in segment_results) else: new_object_manifest = '%s/%s/%s/%s/%s/' % ( quote(seg_container.encode('utf8')), @@ -2221,8 +2225,9 @@ response = self._upload_slo_manifest( conn, results, container, obj, put_headers) res['manifest_response_dict'] = response - new_slo_manifest_paths = { - r['segment_location'] for r in results} + new_slo_manifest_paths.update( + normalize_manifest_path(new_seg['segment_location']) + for new_seg in results) res['large_object'] = True else: res['response_dict'] = ret @@ -2262,11 +2267,10 @@ fp.close() if old_manifest or old_slo_manifest_paths: drs = [] - delobjsmap = {} + delobjsmap = defaultdict(list) if old_manifest: scontainer, sprefix = old_manifest.split('/', 1) sprefix = sprefix.rstrip('/') + '/' - delobjsmap[scontainer] = [] for part in self.list(scontainer, {'prefix': sprefix}): if not part["success"]: raise part["error"] @@ -2278,10 +2282,8 @@ if seg_to_delete in new_slo_manifest_paths: continue scont, sobj = \ - seg_to_delete.split(b'/', 1) - delobjs_cont = delobjsmap.get(scont, []) - delobjs_cont.append(sobj) - delobjsmap[scont] = delobjs_cont + seg_to_delete.split('/', 1) + delobjsmap[scont].append(sobj) del_segs = [] for dscont, dsobjs in delobjsmap.items(): @@ -2514,7 +2516,8 @@ if not options['leave_segments']: try: headers = conn.head_object(container, obj, - headers=_headers) + headers=_headers, + query_string='symlink=get') old_manifest = headers.get('x-object-manifest') if config_true_value(headers.get('x-static-large-object')): query_string = 'multipart-manifest=delete' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/swiftclient/shell.py new/python-swiftclient-3.8.1/swiftclient/shell.py --- old/python-swiftclient-3.7.0/swiftclient/shell.py 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/swiftclient/shell.py 2019-09-13 23:39:55.000000000 +0200 @@ -33,7 +33,8 @@ from time import gmtime, strftime from swiftclient import RequestException -from swiftclient.utils import config_true_value, generate_temp_url, prt_bytes +from swiftclient.utils import config_true_value, generate_temp_url, \ + prt_bytes, JSONableIterable from swiftclient.multithreading import OutputManager from swiftclient.exceptions import ClientException from swiftclient import __version__ as client_version @@ -58,6 +59,7 @@ stderr.write(" Aborted\n") os_exit(2) + st_delete_options = '''[--all] [--leave-segments] [--object-threads <threads>] [--container-threads <threads>] @@ -577,6 +579,8 @@ help='Roll up items with the given delimiter. For containers ' 'only. See OpenStack Swift API documentation for ' 'what this means.') + parser.add_argument('-j', '--json', action='store_true', + help='print listing information in json') parser.add_argument( '-H', '--header', action='append', dest='header', default=[], @@ -615,6 +619,20 @@ else: stats_parts_gen = swift.list(container=container) + if options.get('json', False): + def listing(stats_parts_gen=stats_parts_gen): + for stats in stats_parts_gen: + if stats["success"]: + for item in stats['listing']: + yield item + else: + raise stats["error"] + + json.dump( + JSONableIterable(listing()), output_manager.print_stream, + sort_keys=True, indent=2) + output_manager.print_msg('') + return for stats in stats_parts_gen: if stats["success"]: _print_stats(options, stats, human) @@ -708,7 +726,7 @@ output_manager.error(e.value) -st_post_options = '''[--read-acl <acl>] [--write-acl <acl>] [--sync-to] +st_post_options = '''[--read-acl <acl>] [--write-acl <acl>] [--sync-to <sync-to>] [--sync-key <sync-key>] [--meta <name:value>] [--header <header>] [<container> [<object>]] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/swiftclient/utils.py new/python-swiftclient-3.8.1/swiftclient/utils.py --- old/python-swiftclient-3.7.0/swiftclient/utils.py 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/swiftclient/utils.py 2019-09-13 23:39:55.000000000 +0200 @@ -74,7 +74,7 @@ Swift object. :param path: The full path to the Swift object or prefix if - a prefix-based temporary URL should be generated. Example: + a prefix-based temporary URL should be generated. Example: /v1/AUTH_account/c/o or /v1/AUTH_account/c/prefix. :param seconds: time in seconds or ISO 8601 timestamp. If absolute is False and this is the string representation of an @@ -395,3 +395,33 @@ def n_groups(seq, n): items_per_group = ((len(seq) - 1) // n) + 1 return n_at_a_time(seq, items_per_group) + + +def normalize_manifest_path(path): + if six.PY2 and isinstance(path, six.text_type): + path = path.encode('utf-8') + if path.startswith('/'): + return path[1:] + return path + + +class JSONableIterable(list): + def __init__(self, iterable): + self._iterable = iter(iterable) + try: + self._peeked = next(self._iterable) + self._has_items = True + except StopIteration: + self._peeked = None + self._has_items = False + + def __bool__(self): + return self._has_items + + __nonzero__ = __bool__ + + def __iter__(self): + if self._has_items: + yield self._peeked + for item in self._iterable: + yield item diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/test-requirements.txt new/python-swiftclient-3.8.1/test-requirements.txt --- old/python-swiftclient-3.7.0/test-requirements.txt 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/test-requirements.txt 2019-09-13 23:39:55.000000000 +0200 @@ -3,8 +3,4 @@ coverage!=4.4,>=4.0 # Apache-2.0 keystoneauth1>=3.4.0 # Apache-2.0 mock>=1.2.0 # BSD -oslosphinx>=4.7.0 # Apache-2.0 -sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD stestr>=2.0.0 # Apache-2.0 -reno>=2.5.0 # Apache-2.0 -openstackdocstheme>=1.18.1 # Apache-2.0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/tests/functional/test_swiftclient.py new/python-swiftclient-3.8.1/tests/functional/test_swiftclient.py --- old/python-swiftclient-3.7.0/tests/functional/test_swiftclient.py 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/tests/functional/test_swiftclient.py 2019-09-13 23:39:55.000000000 +0200 @@ -18,6 +18,7 @@ import time from io import BytesIO +import six from six.moves import configparser import swiftclient @@ -46,11 +47,34 @@ config.read(config_file) self.config = config if config.has_section('func_test'): - auth_host = config.get('func_test', 'auth_host') - auth_port = config.getint('func_test', 'auth_port') - auth_ssl = config.getboolean('func_test', 'auth_ssl') - auth_prefix = config.get('func_test', 'auth_prefix') - self.auth_version = config.get('func_test', 'auth_version') + if config.has_option('func_test', 'auth_uri'): + self.auth_url = config.get('func_test', 'auth_uri') + try: + self.auth_version = config.get('func_test', 'auth_version') + except configparser.NoOptionError: + last_piece = self.auth_url.rstrip('/').rsplit('/', 1)[1] + if last_piece.endswith('.0'): + last_piece = last_piece[:-2] + if last_piece in ('1', '2', '3'): + self.auth_version = last_piece + else: + raise + else: + auth_host = config.get('func_test', 'auth_host') + auth_port = config.getint('func_test', 'auth_port') + auth_ssl = config.getboolean('func_test', 'auth_ssl') + auth_prefix = config.get('func_test', 'auth_prefix') + self.auth_version = config.get('func_test', 'auth_version') + self.auth_url = "" + if auth_ssl: + self.auth_url += "https://"; + else: + self.auth_url += "http://"; + self.auth_url += "%s:%s%s" % ( + auth_host, auth_port, auth_prefix) + if self.auth_version == "1": + self.auth_url += 'v1.0' + try: self.account_username = config.get('func_test', 'account_username') @@ -59,15 +83,6 @@ username = config.get('func_test', 'username') self.account_username = "%s:%s" % (account, username) self.password = config.get('func_test', 'password') - self.auth_url = "" - if auth_ssl: - self.auth_url += "https://"; - else: - self.auth_url += "http://"; - self.auth_url += "%s:%s%s" % (auth_host, auth_port, auth_prefix) - if self.auth_version == "1": - self.auth_url += 'v1.0' - else: self.skip_tests = True @@ -432,6 +447,22 @@ self.assertEqual('45.67', headers.get('x-object-meta-float')) self.assertEqual('False', headers.get('x-object-meta-bool')) + def test_post_object_unicode_header_name(self): + self.conn.post_object(self.containername, + self.objectname, + {u'x-object-meta-\U0001f44d': u'\U0001f44d'}) + + # Note that we can't actually read this header back on py3; see + # https://bugs.python.org/issue37093 + # We'll have to settle for just testing that the POST doesn't blow up + # with a UnicodeDecodeError + if six.PY2: + headers = self.conn.head_object( + self.containername, self.objectname) + self.assertIn(u'x-object-meta-\U0001f44d', headers) + self.assertEqual(u'\U0001f44d', + headers.get(u'x-object-meta-\U0001f44d')) + def test_copy_object(self): self.conn.put_object( self.containername, self.objectname, self.test_data) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/tests/unit/test_service.py new/python-swiftclient-3.8.1/tests/unit/test_service.py --- old/python-swiftclient-3.7.0/tests/unit/test_service.py 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/tests/unit/test_service.py 2019-09-13 23:39:55.000000000 +0200 @@ -312,8 +312,8 @@ s = SwiftService() r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) - mock_conn.head_object.assert_called_once_with('test_c', 'test_o', - headers={}) + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={}, headers={} @@ -335,7 +335,8 @@ r = s._delete_object(mock_conn, 'test_c', 'test_o', opt_c, mock_q) mock_conn.head_object.assert_called_once_with( - 'test_c', 'test_o', headers={'Skip-Middleware': 'Test'}) + 'test_c', 'test_o', headers={'Skip-Middleware': 'Test'}, + query_string='symlink=get') mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={}, headers={'Skip-Middleware': 'Test'} @@ -362,8 +363,8 @@ r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) after = time.time() - mock_conn.head_object.assert_called_once_with('test_c', 'test_o', - headers={}) + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={}, headers={} @@ -389,8 +390,8 @@ s = SwiftService() r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) - mock_conn.head_object.assert_called_once_with('test_c', 'test_o', - headers={}) + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string='multipart-manifest=delete', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/tests/unit/test_shell.py new/python-swiftclient-3.8.1/tests/unit/test_shell.py --- old/python-swiftclient-3.7.0/tests/unit/test_shell.py 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/tests/unit/test_shell.py 2019-09-13 23:39:55.000000000 +0200 @@ -298,6 +298,26 @@ headers={'Skip-Middleware': 'Test'})]) @mock.patch('swiftclient.service.Connection') + def test_list_json(self, connection): + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, [{'name': u'\u263A', 'some-custom-key': 'and value'}]], + [None, []], + ] + + argv = ["", "list", "--json"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, headers={}), + mock.call(marker='container', prefix=None, headers={})] + connection.return_value.get_account.assert_has_calls(calls) + + listing = [{'name': 'container'}, + {'name': u'\u263A', 'some-custom-key': 'and value'}] + expected = json.dumps(listing, sort_keys=True, indent=2) + '\n' + self.assertEqual(output.out, expected) + + @mock.patch('swiftclient.service.Connection') def test_list_account(self, connection): # Test account listing connection.return_value.get_account.side_effect = [ @@ -799,11 +819,11 @@ response_dict={}) expected_delete_calls = [ mock.call( - b'container1', b'old_seg1', + 'container1', 'old_seg1', response_dict={} ), mock.call( - b'container2', b'old_seg2', + 'container2', 'old_seg2', response_dict={} ) ] @@ -813,6 +833,35 @@ ) @mock.patch('swiftclient.service.Connection') + def test_upload_over_symlink_to_slo(self, connection): + # Upload delete existing segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', + 'content-location': '/v1/a/c/manifest', + 'content-length': '2'}, + ] + connection.return_value.get_object.return_value = ( + {'content-location': '/v1/a/c/manifest'}, + b'[{"name": "container1/old_seg1"},' + b' {"name": "container2/old_seg2"}]' + ) + connection.return_value.put_object.return_value = EMPTY_ETAG + connection.return_value.delete_object.return_value = None + argv = ["", "upload", "container", self.tmpfile] + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + self.assertEqual([], connection.return_value.delete_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') def test_upload_leave_slo_segments(self, connection): # Test upload overwriting a manifest respects --leave-segments connection.return_value.head_container.return_value = { @@ -835,6 +884,46 @@ self.assertFalse(connection.return_value.delete_object.mock_calls) @mock.patch('swiftclient.service.Connection') + def test_reupload_leaves_slo_segments(self, connection): + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + mtime = '{:.6f}'.format(os.path.getmtime(self.tmpfile)) + expected_segments = [ + 'container_segments/{}/slo/{}/20/10/{:08d}'.format( + self.tmpfile[1:], mtime, i) + for i in range(2) + ] + + # Test re-upload overwriting a manifest doesn't remove + # segments it just wrote + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile, + "--use-slo", "-S", "10"] + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', # For the upload call + 'content-length': '20'}] + connection.return_value.get_object.return_value = ( + {}, + # we've already *got* the expected manifest! + json.dumps([ + {'name': seg} for seg in expected_segments + ]).encode('ascii') + ) + connection.return_value.put_object.return_value = ( + 'd41d8cd98f00b204e9800998ecf8427e') + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile[1:], # drop leading / + mock.ANY, + headers={'x-object-meta-mtime': mtime}, + query_string='multipart-manifest=put', + response_dict={}) + self.assertFalse(connection.return_value.delete_object.mock_calls) + + @mock.patch('swiftclient.service.Connection') def test_upload_delete_dlo_segments(self, connection): # Upload delete existing segments connection.return_value.head_container.return_value = { @@ -1822,7 +1911,7 @@ argv = ["", "tempurl", "GET", "60", '/v1/a/c', "secret_key", "--absolute", '--prefix-based'] with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) + swiftclient.shell.main(argv) self.assertEqual(expected, output.err, 'Expected %r but got %r for path %r' % (expected, output.err, '/v1/a/c')) @@ -1832,7 +1921,7 @@ argv = ["", "tempurl", "GET", bad_time, '/v1/a/c/o', "secret_key", "--absolute"] with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) + swiftclient.shell.main(argv) self.assertEqual(expected, output.err, 'Expected %r but got %r for time %r' % (expected, output.err, bad_time)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/tests/unit/test_utils.py new/python-swiftclient-3.8.1/tests/unit/test_utils.py --- old/python-swiftclient-3.7.0/tests/unit/test_utils.py 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/tests/unit/test_utils.py 2019-09-13 23:39:55.000000000 +0200 @@ -14,6 +14,7 @@ # limitations under the License. import gzip +import json import unittest import mock import six @@ -638,3 +639,41 @@ {'content-encoding': 'gzip'}, buf.getvalue()) self.assertEqual({'test': u'\u2603'}, result) + + +class JSONTracker(object): + def __init__(self, data): + self.data = data + self.calls = [] + + def __iter__(self): + for item in self.data: + self.calls.append(('read', item)) + yield item + + def write(self, s): + self.calls.append(('write', s)) + + +class TestJSONableIterable(unittest.TestCase): + def test_json_dump_iterencodes(self): + t = JSONTracker([1, 'fish', 2, 'fish']) + json.dump(u.JSONableIterable(t), t) + self.assertEqual(t.calls, [ + ('read', 1), + ('write', '[1'), + ('read', 'fish'), + ('write', ', "fish"'), + ('read', 2), + ('write', ', 2'), + ('read', 'fish'), + ('write', ', "fish"'), + ('write', ']'), + ]) + + def test_json_dump_empty_iter(self): + t = JSONTracker([]) + json.dump(u.JSONableIterable(t), t) + self.assertEqual(t.calls, [ + ('write', '[]'), + ]) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/tests/unit/utils.py new/python-swiftclient-3.8.1/tests/unit/utils.py --- old/python-swiftclient-3.7.0/tests/unit/utils.py 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/tests/unit/utils.py 2019-09-13 23:39:55.000000000 +0200 @@ -20,6 +20,7 @@ import unittest import mock import six +import os from six.moves import reload_module from six.moves.urllib.parse import urlparse, ParseResult from swiftclient import client as c @@ -212,7 +213,19 @@ # won't cover the references to sys.stdout/sys.stderr in # swiftclient.multithreading self.capture_output = CaptureOutput() - self.capture_output.__enter__() + if 'SWIFTCLIENT_DEBUG' not in os.environ: + self.capture_output.__enter__() + self.addCleanup(self.capture_output.__exit__) + + # since we're going to steal all stderr output globally; we should + # give the developer an escape hatch or risk scorn + def blowup_but_with_the_helpful(*args, **kwargs): + raise Exception( + "You tried to enter a debugger while stderr is " + "patched, you need to set SWIFTCLIENT_DEBUG=1 " + "and try again") + import pdb + pdb.set_trace = blowup_but_with_the_helpful def fake_http_connection(*args, **kwargs): self.validateMockedRequestsConsumed() @@ -391,7 +404,6 @@ # un-hygienic mocking on the swiftclient.client module; which may lead # to some unfortunate test order dependency bugs by way of the broken # window theory if any other modules are similarly patched - self.capture_output.__exit__() reload_module(c) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/python-swiftclient-3.7.0/tox.ini new/python-swiftclient-3.8.1/tox.ini --- old/python-swiftclient-3.7.0/tox.ini 2019-03-05 14:10:18.000000000 +0100 +++ new/python-swiftclient-3.8.1/tox.ini 2019-09-13 23:39:55.000000000 +0200 @@ -1,5 +1,5 @@ [tox] -envlist = py37,py36,py35,py27,pypy,pep8 +envlist = py27,py37,pypy,pep8 minversion = 2.0 skipsdist = True @@ -65,8 +65,10 @@ [testenv:docs] basepython = python3 +usedevelop = False +deps = -r{toxinidir}/doc/requirements.txt commands= - python setup.py build_sphinx + python setup.py build_sphinx -W [flake8] # it's not a bug that we aren't using all of hacking, ignore: @@ -77,7 +79,11 @@ # H403: multi line docstrings should end on a new line # H404: multi line docstring should start without a leading new line # H405: multi line docstring summary not separated with an empty line -ignore = E731,H101,H301,H306,H401,H403,H404,H405 +# W504: line break after binary operator +ignore = H101,H301,H306,H401,H403,H404,H405,W504 +# H106: Don’t put vim configuration in source files +# H203: Use assertIs(Not)None to check for None +enable-extensions=H106,H203 show-source = True exclude = .venv,.tox,dist,doc,*egg @@ -93,6 +99,8 @@ [testenv:releasenotes] basepython = python3 +usedevelop = False +deps = -r{toxinidir}/doc/requirements.txt commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html [testenv:lower-constraints] @@ -101,3 +109,12 @@ -c{toxinidir}/lower-constraints.txt -r{toxinidir}/test-requirements.txt .[keystone] + +[testenv:pdf-docs] +basepython = python3 +deps = {[testenv:docs]deps} +whitelist_externals = + make +commands = + sphinx-build -W -b latex doc/source doc/build/pdf + make -C doc/build/pdf
