This is an automated email from the ASF dual-hosted git repository. tomaz pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit ad084ac7e21d007cb655f43e483d32a1c427fd2e Author: Tomaz Muraus <[email protected]> AuthorDate: Sat Dec 19 11:57:28 2020 +0100 Add new libcloud.common.base.ALLOW_PATH_DOUBLE_SLASHES which default to False for backward compatibility reasons. When set to True, Libcloud won't perform any request url path sanitization and will allow paths with duplicated slashes (e.g. /my-bucket//foo/bar.txt). This may come handy in scenarios such as S3 where duplicated slashes are considered valid. --- CHANGES.rst | 18 ++++++++++++++++++ libcloud/common/base.py | 22 ++++++++++++++++++++++ libcloud/test/test_connection.py | 22 ++++++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 394439a..68a889c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -161,6 +161,24 @@ Storage (GITHUB-1528) [Tomaz Muraus - @Kami] +- Add new ``libcloud.common.base.ALLOW_PATH_DOUBLE_SLASHES`` module level + variable. + + When this value is set to ``True`` (defaults to ``False`` for backward + compatibility reasons), Libcloud won't try to sanitize the URL path and + remove any double slashes. + + In most cases, this won't matter and sanitzing double slashes is a safer + default, but in some cases such as S3, where double slashes can be a valid + path (e.g. ``/my-bucket//path1/file.txt``), this option may come handy. + + When this variable is set to ``True``, behavior is also consistent with + Libcloud versions prior to v2.0.0. + + Reported by Jonathan Hanson - @triplepoint. + (GITHUB-1529) + [Tomaz Muraus - @Kami] + DNS ~~~ diff --git a/libcloud/common/base.py b/libcloud/common/base.py index 51bf9a3..48b0e22 100644 --- a/libcloud/common/base.py +++ b/libcloud/common/base.py @@ -59,6 +59,13 @@ __all__ = [ # Module level variable indicates if the failed HTTP requests should be retried RETRY_FAILED_HTTP_REQUESTS = False +# Set to True to allow double slashes in the URL path. This way +# morph_action_hook() won't strip potentially double slashes in the URLs. +# This is to support scenarios such as this one - +# https://github.com/apache/libcloud/issues/1529. +# We default it to False for backward compatibility reasons. +ALLOW_PATH_DOUBLE_SLASHES = False + class LazyObject(object): """An object that doesn't get initialized until accessed.""" @@ -653,8 +660,23 @@ class Connection(object): return response def morph_action_hook(self, action): + """ + Here we strip any duplicated leading or traling slashes to + prevent typos and other issues where some APIs don't correctly + handle double slashes. + + Keep in mind that in some situations, "/" is a vallid path name + so we have a module flag which disables this behavior + (https://github.com/apache/libcloud/issues/1529). + """ + if ALLOW_PATH_DOUBLE_SLASHES: + # Special case to support scenarios where double slashes are + # valid - e.g. for S3 paths - /bucket//path1/path2.txt + return self.request_path + action + url = urlparse.urljoin(self.request_path.lstrip('/').rstrip('/') + '/', action.lstrip('/')) + if not url.startswith('/'): return '/' + url else: diff --git a/libcloud/test/test_connection.py b/libcloud/test/test_connection.py index f00177b..780c1aa 100644 --- a/libcloud/test/test_connection.py +++ b/libcloud/test/test_connection.py @@ -23,6 +23,8 @@ from mock import Mock, patch import requests_mock +import libcloud.common.base + from libcloud.test import unittest from libcloud.common.base import Connection, CertificateConnection from libcloud.http import LibcloudBaseConnection @@ -209,6 +211,26 @@ class BaseConnectionClassTestCase(unittest.TestCase): self.assertEqual(conn.morph_action_hook('/test'), '/v1/test') self.assertEqual(conn.morph_action_hook('test'), '/v1/test') + conn.request_path = '/a' + self.assertEqual(conn.morph_action_hook('//b/c.txt'), '/a/b/c.txt') + + conn.request_path = '/b' + self.assertEqual(conn.morph_action_hook('/foo//'), '/b/foo/') + + libcloud.common.base.ALLOW_PATH_DOUBLE_SLASHES = True + + conn.request_path = '/' + self.assertEqual(conn.morph_action_hook('/'), '//') + + conn.request_path = '' + self.assertEqual(conn.morph_action_hook('/'), '/') + + conn.request_path = '/a' + self.assertEqual(conn.morph_action_hook('//b/c.txt'), '/a//b/c.txt') + + conn.request_path = '/b' + self.assertEqual(conn.morph_action_hook('/foo//'), '/b/foo//') + def test_connect_with_prefix(self): """ Test that a connection with a base path (e.g. /v1/) will
