This is an automated email from the ASF dual-hosted git repository.
clewolff pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/libcloud.git
The following commit(s) were added to refs/heads/trunk by this push:
new 38693a4 Add integration tests for Azure Storage driver (#1572)
38693a4 is described below
commit 38693a4775505364707419447fb9eb788f8f04e9
Author: Clemens Wolff <[email protected]>
AuthorDate: Sun May 2 10:31:08 2021 -0400
Add integration tests for Azure Storage driver (#1572)
* Fix build
* Namespace compute integration tests
* Add integration tests for Azure Storage driver
---
.github/workflows/main.yml | 38 +++
integration/README.rst | 20 --
integration/compute/README.rst | 20 ++
integration/{driver => compute}/__init__.py | 0
integration/{ => compute}/__main__.py | 4 +-
integration/{ => compute}/api/__init__.py | 0
integration/{ => compute}/api/__main__.py | 0
integration/{ => compute}/api/data.py | 0
integration/{ => compute}/api/routes.py | 4 +-
integration/{ => compute}/api/util.py | 2 +-
integration/{ => compute}/config.py | 0
integration/{ => compute}/driver/__init__.py | 0
integration/{ => compute}/driver/test.py | 0
integration/{ => compute}/requirements.txt | 0
integration/storage/README.rst | 18 ++
integration/{driver => storage}/__init__.py | 0
integration/{api => storage}/__main__.py | 11 +-
integration/storage/base.py | 386 +++++++++++++++++++++++++++
integration/storage/requirements.txt | 5 +
integration/storage/test_azure_blobs.py | 177 ++++++++++++
tox.ini | 26 +-
21 files changed, 676 insertions(+), 35 deletions(-)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index f9d2750..585bdb7 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -57,6 +57,7 @@ jobs:
- name: Install OS / deb dependencies
run: |
+ sudo DEBIAN_FRONTEND=noninteractive apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq gcc
libvirt-dev
- name: Install Python Dependencies
@@ -67,6 +68,40 @@ jobs:
run: |
tox -e py${{ matrix.python_version }}
+ integ_tests:
+ name: Run Integration Tests
+ runs-on: ubuntu-latest
+
+ # needs: pre_job
+ # if: ${{ needs.pre_job.outputs.should_skip == 'false' || github.ref ==
'refs/heads/trunk' }}
+
+ strategy:
+ matrix:
+ python_version: [3.7]
+
+ steps:
+ - uses: actions/checkout@master
+ with:
+ fetch-depth: 1
+
+ - name: Use Python ${{ matrix.python_version }}
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python_version }}
+
+ - name: Install OS / deb dependencies
+ run: |
+ sudo DEBIAN_FRONTEND=noninteractive apt-get update
+ sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq gcc
libvirt-dev
+
+ - name: Install Python Dependencies
+ run: |
+ pip install "tox==3.20.1"
+
+ - name: Run tox target
+ run: |
+ tox -e integration-storage
+
code_coverage:
name: Generate Code Coverage
runs-on: ubuntu-latest
@@ -90,6 +125,7 @@ jobs:
- name: Install OS / deb dependencies
run: |
+ sudo DEBIAN_FRONTEND=noninteractive apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq graphviz gcc
libvirt-dev
- name: Install Python Dependencies
@@ -123,6 +159,7 @@ jobs:
- name: Install OS / deb dependencies
run: |
+ sudo DEBIAN_FRONTEND=noninteractive apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq graphviz gcc
libvirt-dev
- name: Install Python Dependencies
@@ -161,6 +198,7 @@ jobs:
- name: Install OS / deb dependencies
run: |
+ sudo DEBIAN_FRONTEND=noninteractive apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq graphviz gcc
libvirt-dev
- name: Install Python Dependencies
diff --git a/integration/README.rst b/integration/README.rst
deleted file mode 100644
index ed61b5a..0000000
--- a/integration/README.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-Integration Test Module
-=======================
-
-This test suite is for running a live API endpoint and testing the
apache-libcloud functionality as a full integration test
-
-Running the API service
------------------------
-
-.. code-block:: bash
-
- pip install -r integration/requirements.txt
- python -m integration.api
-
-Running the tests
------------------
-
-.. code-block:: bash
-
- python -m integration
-
diff --git a/integration/compute/README.rst b/integration/compute/README.rst
new file mode 100644
index 0000000..7c010f1
--- /dev/null
+++ b/integration/compute/README.rst
@@ -0,0 +1,20 @@
+Compute Integration Test Module
+===============================
+
+This test suite is for running a live API endpoint and testing the
apache-libcloud compute functionality as a full integration test
+
+Running the API service
+-----------------------
+
+.. code-block:: bash
+
+ pip install -r integration/compute/requirements.txt
+ python -m integration.compute.api
+
+Running the tests
+-----------------
+
+.. code-block:: bash
+
+ python -m integration.compute
+
diff --git a/integration/driver/__init__.py b/integration/compute/__init__.py
similarity index 100%
copy from integration/driver/__init__.py
copy to integration/compute/__init__.py
diff --git a/integration/__main__.py b/integration/compute/__main__.py
similarity index 94%
rename from integration/__main__.py
rename to integration/compute/__main__.py
index 1d04b00..21155bb 100644
--- a/integration/__main__.py
+++ b/integration/compute/__main__.py
@@ -16,9 +16,9 @@
import unittest
import sys
-from integration.driver.test import TestNodeDriver
+from integration.compute.driver.test import TestNodeDriver
-from integration.api.data import NODES, REPORT_DATA
+from integration.compute.api.data import NODES, REPORT_DATA
class IntegrationTest(unittest.TestCase):
diff --git a/integration/api/__init__.py b/integration/compute/api/__init__.py
similarity index 100%
rename from integration/api/__init__.py
rename to integration/compute/api/__init__.py
diff --git a/integration/api/__main__.py b/integration/compute/api/__main__.py
similarity index 100%
copy from integration/api/__main__.py
copy to integration/compute/api/__main__.py
diff --git a/integration/api/data.py b/integration/compute/api/data.py
similarity index 100%
rename from integration/api/data.py
rename to integration/compute/api/data.py
diff --git a/integration/api/routes.py b/integration/compute/api/routes.py
similarity index 90%
rename from integration/api/routes.py
rename to integration/compute/api/routes.py
index 05fd0ec..b986ea3 100644
--- a/integration/api/routes.py
+++ b/integration/compute/api/routes.py
@@ -17,8 +17,8 @@ import json
from bottle import route
-from integration.api.data import NODES, REPORT_DATA
-from integration.api.util import secure
+from integration.compute.api.data import NODES, REPORT_DATA
+from integration.compute.api.util import secure
@route('/compute/nodes', method='GET')
diff --git a/integration/api/util.py b/integration/compute/api/util.py
similarity index 95%
rename from integration/api/util.py
rename to integration/compute/api/util.py
index 679ff98..20c2e0e 100644
--- a/integration/api/util.py
+++ b/integration/compute/api/util.py
@@ -16,7 +16,7 @@
from bottle import request
from functools import wraps
-from integration.config import EXPECTED_AUTH
+from integration.compute.config import EXPECTED_AUTH
def secure(f):
diff --git a/integration/config.py b/integration/compute/config.py
similarity index 100%
rename from integration/config.py
rename to integration/compute/config.py
diff --git a/integration/driver/__init__.py
b/integration/compute/driver/__init__.py
similarity index 100%
copy from integration/driver/__init__.py
copy to integration/compute/driver/__init__.py
diff --git a/integration/driver/test.py b/integration/compute/driver/test.py
similarity index 100%
rename from integration/driver/test.py
rename to integration/compute/driver/test.py
diff --git a/integration/requirements.txt b/integration/compute/requirements.txt
similarity index 100%
rename from integration/requirements.txt
rename to integration/compute/requirements.txt
diff --git a/integration/storage/README.rst b/integration/storage/README.rst
new file mode 100644
index 0000000..91d6ca8
--- /dev/null
+++ b/integration/storage/README.rst
@@ -0,0 +1,18 @@
+Storage Integration Test Module
+===============================
+
+This test suite is for validating the apache-libcloud storage functionality
against live storage backends.
+
+Setting up the test suite
+-------------------------
+
+.. code-block:: bash
+
+ pip install -r integration/storage/requirements.txt
+
+Running the tests
+-----------------
+
+.. code-block:: bash
+
+ python -m integration.storage
diff --git a/integration/driver/__init__.py b/integration/storage/__init__.py
similarity index 100%
rename from integration/driver/__init__.py
rename to integration/storage/__init__.py
diff --git a/integration/api/__main__.py b/integration/storage/__main__.py
similarity index 77%
rename from integration/api/__main__.py
rename to integration/storage/__main__.py
index 85d7dc7..f4fba46 100644
--- a/integration/api/__main__.py
+++ b/integration/storage/__main__.py
@@ -13,9 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from bottle import run
+import os
+import sys
+import unittest
-import integration.api.routes # noqa
if __name__ == '__main__':
- run(host='localhost', port=9898)
+ loader = unittest.TestLoader()
+ tests = loader.discover(os.path.dirname(__file__))
+ runner = unittest.runner.TextTestRunner()
+ result = runner.run(tests)
+ sys.exit(len(result.errors))
diff --git a/integration/storage/base.py b/integration/storage/base.py
new file mode 100644
index 0000000..119e8b1
--- /dev/null
+++ b/integration/storage/base.py
@@ -0,0 +1,386 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import gzip
+import io
+import os
+import random
+import re
+import socket
+import string
+import sys
+import tempfile
+import time
+import unittest
+
+import requests
+
+try:
+ import docker
+except ImportError:
+ docker = None
+
+from libcloud.common.types import LibcloudError
+from libcloud.storage import providers, types
+
+
+MB = 1024 * 1024
+
+
+class Integration:
+ class TestBase(unittest.TestCase):
+ provider = None
+ account = None
+ secret = None
+
+ def setUp(self):
+ for required in 'provider', 'account', 'secret':
+ value = getattr(self, required, None)
+ if value is None:
+ raise unittest.SkipTest('config {} not
set'.format(required))
+
+ kwargs = {'key': self.account, 'secret': self.secret}
+
+ for optional in 'host', 'port', 'secure':
+ value = getattr(self, optional, None)
+ if value is not None:
+ kwargs[optional] = value
+
+ driver_class = providers.get_driver(self.provider)
+ self.driver = driver_class(**kwargs)
+
+ def tearDown(self):
+ for container in self.driver.list_containers():
+ for obj in container.list_objects():
+ try:
+ obj.delete()
+ except LibcloudError as ex:
+ print(
+ 'Unable to delete object {} in container {}: {}.'
+ 'Delete it manually.'
+ .format(obj.name, container.name, ex),
+ file=sys.stderr
+ )
+
+ try:
+ container.delete()
+ except LibcloudError as ex:
+ print(
+ 'Unable to delete container {}: {}.'
+ 'Delete it manually.'
+ .format(container.name, ex),
+ file=sys.stderr
+ )
+
+ def test_containers(self):
+ # make a new container
+ container_name = random_container_name()
+ container = self.driver.create_container(container_name)
+ self.assertEqual(container.name, container_name)
+ container = self.driver.get_container(container_name)
+ self.assertEqual(container.name, container_name)
+
+ # check that an existing container can't be re-created
+ with self.assertRaises(types.ContainerAlreadyExistsError):
+ self.driver.create_container(container_name)
+
+ # check that the new container can be listed
+ containers = self.driver.list_containers()
+ self.assertEqual([c.name for c in containers], [container_name])
+
+ # delete the container
+ self.driver.delete_container(container)
+
+ # check that a deleted container can't be looked up
+ with self.assertRaises(types.ContainerDoesNotExistError):
+ self.driver.get_container(container_name)
+
+ # check that the container is deleted
+ containers = self.driver.list_containers()
+ self.assertEqual([c.name for c in containers], [])
+
+ def _test_objects(self, do_upload, do_download, size=1 * MB):
+ content = os.urandom(size)
+ blob_name = 'testblob'
+ container = self.driver.create_container(random_container_name())
+
+ # upload a file
+ obj = do_upload(container, blob_name, content)
+ self.assertEqual(obj.name, blob_name)
+ obj = self.driver.get_object(container.name, blob_name)
+
+ # check that the file can be listed
+ blobs = self.driver.list_container_objects(container)
+ self.assertEqual([blob.name for blob in blobs], [blob_name])
+
+ # upload another file and check it's excluded in prefix listing
+ do_upload(container, blob_name[::-1], content[::-1])
+ blobs = self.driver.list_container_objects(
+ container, prefix=blob_name[0:3]
+ )
+ self.assertEqual([blob.name for blob in blobs], [blob_name])
+
+ # check that the file can be read back
+ self.assertEqual(do_download(obj), content)
+
+ # delete the file
+ self.driver.delete_object(obj)
+
+ # check that a missing file can't be deleted or looked up
+ with self.assertRaises(types.ObjectDoesNotExistError):
+ self.driver.delete_object(obj)
+ with self.assertRaises(types.ObjectDoesNotExistError):
+ self.driver.get_object(container.name, blob_name)
+
+ # check that the file is deleted
+ blobs = self.driver.list_container_objects(container)
+ self.assertEqual([blob.name for blob in blobs], [blob_name[::-1]])
+
+ def test_objects(self, size=1 * MB):
+ def do_upload(container, blob_name, content):
+ infile = self._create_tempfile(content=content)
+ return self.driver.upload_object(infile, container, blob_name)
+
+ def do_download(obj):
+ outfile = self._create_tempfile()
+ self.driver.download_object(obj, outfile,
overwrite_existing=True)
+ with open(outfile, 'rb') as fobj:
+ return fobj.read()
+
+ self._test_objects(do_upload, do_download, size)
+
+ def test_objects_range_downloads(self):
+ blob_name = 'testblob-range'
+ content = b'0123456789'
+ container = self.driver.create_container(random_container_name())
+
+ obj = self.driver.upload_object(
+ self._create_tempfile(content=content),
+ container,
+ blob_name
+ )
+ self.assertEqual(obj.name, blob_name)
+ self.assertEqual(obj.size, len(content))
+
+ obj = self.driver.get_object(container.name, blob_name)
+ self.assertEqual(obj.name, blob_name)
+ self.assertEqual(obj.size, len(content))
+
+ values = [
+ {'start_bytes': 0, 'end_bytes': 1, 'expected_content': b'0'},
+ {'start_bytes': 1, 'end_bytes': 5, 'expected_content':
b'1234'},
+ {'start_bytes': 5, 'end_bytes': None, 'expected_content':
b'56789'},
+ {'start_bytes': 5, 'end_bytes': len(content),
'expected_content': b'56789'},
+ {'start_bytes': 0, 'end_bytes': None, 'expected_content':
b'0123456789'},
+ {'start_bytes': 0, 'end_bytes': len(content),
'expected_content': b'0123456789'},
+ ]
+
+ for value in values:
+ # 1. download_object_range
+ start_bytes = value['start_bytes']
+ end_bytes = value['end_bytes']
+ outfile = self._create_tempfile()
+
+ result = self.driver.download_object_range(
+ obj,
+ outfile,
+ start_bytes=start_bytes,
+ end_bytes=end_bytes,
+ overwrite_existing=True,
+ )
+ self.assertTrue(result)
+
+ with open(outfile, 'rb') as fobj:
+ downloaded_content = fobj.read()
+
+ if end_bytes is not None:
+ expected_content = content[start_bytes:end_bytes]
+ else:
+ expected_content = content[start_bytes:]
+
+ msg = 'Expected "%s", got "%s" for values: %s' % (
+ expected_content, downloaded_content, str(value)
+ )
+ self.assertEqual(downloaded_content, expected_content, msg)
+ self.assertEqual(downloaded_content,
value['expected_content'], msg)
+
+ # 2. download_object_range_as_stream
+ downloaded_content = read_stream(
+ self.driver.download_object_range_as_stream(
+ obj, start_bytes=start_bytes, end_bytes=end_bytes
+ )
+ )
+ self.assertEqual(downloaded_content, expected_content)
+
+ @unittest.skipUnless(os.getenv('LARGE_FILE_SIZE_MB'), 'config not set')
+ def test_objects_large(self):
+ size = int(float(os.environ['LARGE_FILE_SIZE_MB']) * MB)
+ self.test_objects(size)
+
+ def test_objects_stream_io(self):
+ def do_upload(container, blob_name, content):
+ content = io.BytesIO(content)
+ return self.driver.upload_object_via_stream(content,
container, blob_name)
+
+ def do_download(obj):
+ return read_stream(self.driver.download_object_as_stream(obj))
+
+ self._test_objects(do_upload, do_download)
+
+ def test_objects_stream_iterable(self):
+ def do_upload(container, blob_name, content):
+ content = iter([content[i:i + 1] for i in range(len(content))])
+ return self.driver.upload_object_via_stream(content,
container, blob_name)
+
+ def do_download(obj):
+ return read_stream(self.driver.download_object_as_stream(obj))
+
+ self._test_objects(do_upload, do_download)
+
+ def test_upload_via_stream_with_content_encoding(self):
+ object_name = 'content_encoding.gz'
+ content = gzip.compress(os.urandom(MB // 100))
+ container = self.driver.create_container(random_container_name())
+ self.driver.upload_object_via_stream(
+ iter(content),
+ container,
+ object_name,
+ headers={'Content-Encoding': 'gzip'},
+ )
+
+ obj = self.driver.get_object(container.name, object_name)
+
+ self.assertEqual(obj.extra.get('content_encoding'), 'gzip')
+
+ def test_cdn_url(self):
+ content = os.urandom(MB // 100)
+ container = self.driver.create_container(random_container_name())
+ obj = self.driver.upload_object_via_stream(iter(content),
container, 'cdn')
+
+ response = requests.get(self.driver.get_object_cdn_url(obj))
+ response.raise_for_status()
+
+ self.assertEqual(response.content, content)
+
+ def _create_tempfile(self, prefix='', content=b''):
+ fobj, path = tempfile.mkstemp(prefix=prefix, text=False)
+ os.write(fobj, content)
+ os.close(fobj)
+ self.addCleanup(os.remove, path)
+ return path
+
+ class ContainerTestBase(TestBase):
+ image = None
+ version = 'latest'
+ environment = {}
+ ready_message = None
+
+ host = 'localhost'
+ port = None
+ secure = False
+
+ client = None
+ container = None
+ verbose = False
+
+ @classmethod
+ def setUpClass(cls):
+ if docker is None:
+ raise unittest.SkipTest('missing docker library')
+
+ try:
+ cls.client = docker.from_env()
+ except docker.errors.DockerException:
+ raise unittest.SkipTest('unable to create docker client')
+
+ for required in 'image', 'port':
+ value = getattr(cls, required, None)
+ if value is None:
+ raise unittest.SkipTest('config {} not
set'.format(required))
+
+ cls.container = cls.client.containers.run(
+ '{}:{}'.format(cls.image, cls.version),
+ detach=True,
+ auto_remove=True,
+ ports={cls.port: cls.port},
+ environment=cls.environment,
+ )
+
+ wait_for(cls.port, cls.host)
+
+ container_ready = cls.ready_message is None
+
+ while not container_ready:
+ time.sleep(1)
+
+ container_ready = any(
+ cls.ready_message in line
+ for line in cls.container.logs().splitlines()
+ )
+
+ @classmethod
+ def tearDownClass(cls):
+ if cls.verbose:
+ for line in cls.container.logs().splitlines():
+ print(line)
+
+ try:
+ cls.container.kill()
+ except docker.errors.DockerException as ex:
+ print(
+ 'Unable to terminate docker container {}: {}.'
+ 'Stop it manually.'
+ .format(cls.container.short_id, ex),
+ file=sys.stderr
+ )
+
+
+def wait_for(port, host='localhost', timeout=10):
+ start_time = time.perf_counter()
+
+ while True:
+ try:
+ with socket.create_connection((host, port), timeout=timeout):
+ break
+ except OSError as ex:
+ if time.perf_counter() - start_time >= timeout:
+ raise TimeoutError(
+ 'Waited too long for the port {} on host {} to start
accepting '
+ 'connections.'.format(port, host)
+ ) from ex
+
+ time.sleep(1)
+
+
+def random_string(length, alphabet=string.ascii_lowercase + string.digits):
+ return ''.join(random.choice(alphabet) for _ in range(length))
+
+
+def random_container_name(prefix='test'):
+ max_length = 63
+ suffix = random_string(max_length)
+ name = prefix + suffix
+ name = re.sub('[^a-z0-9-]', '-', name)
+ name = re.sub('-+', '-', name)
+ name = name[:max_length]
+ name = name.lower()
+ return name
+
+
+def read_stream(stream):
+ buffer = io.BytesIO()
+ buffer.writelines(stream)
+ buffer.seek(0)
+ return buffer.read()
diff --git a/integration/storage/requirements.txt
b/integration/storage/requirements.txt
new file mode 100644
index 0000000..cbdd599
--- /dev/null
+++ b/integration/storage/requirements.txt
@@ -0,0 +1,5 @@
+azure-identity
+azure-mgmt-resource
+azure-mgmt-storage
+docker
+requests
diff --git a/integration/storage/test_azure_blobs.py
b/integration/storage/test_azure_blobs.py
new file mode 100644
index 0000000..08a9185
--- /dev/null
+++ b/integration/storage/test_azure_blobs.py
@@ -0,0 +1,177 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the 'License'); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import os
+import string
+import sys
+import unittest
+
+try:
+ from azure import identity
+ from azure.mgmt import resource
+ from azure.mgmt import storage
+ from azure.mgmt.resource.resources import models as resource_models
+ from azure.mgmt.storage import models as storage_models
+except ImportError:
+ identity = resource = storage = resource_models = storage_models = None
+
+from integration.storage.base import Integration, random_string
+
+DEFAULT_TIMEOUT_SECONDS = 300
+DEFAULT_AZURE_LOCATION = 'EastUS2'
+MAX_STORAGE_ACCOUNT_NAME_LENGTH = 24
+
+
+class AzuriteStorageTest(Integration.ContainerTestBase):
+ provider = 'azure_blobs'
+
+ account = 'devstoreaccount1'
+ secret =
'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=='
+
+ image = 'arafato/azurite'
+ port = 10000
+ environment = {'executable': 'blob'}
+ ready_message = b'Azure Blob Storage Emulator listening'
+
+ has_sas_support = False
+
+ def test_cdn_url(self):
+ if not self.has_sas_support:
+ self.skipTest('Storage backend has no account SAS support')
+
+
+class AzuriteV3StorageTest(AzuriteStorageTest):
+ image = 'mcr.microsoft.com/azure-storage/azurite'
+ ready_message = b'Azurite Blob service is successfully listening'
+
+ has_sas_support = True
+
+ def test_upload_via_stream_with_content_encoding(self):
+ self.skipTest('Possible bug in AzuriteV3, see
https://github.com/Azure/Azurite/issues/629')
+
+
+class IotedgeStorageTest(Integration.ContainerTestBase):
+ provider = 'azure_blobs'
+
+ account = random_string(10, string.ascii_lowercase)
+ secret =
base64.b64encode(random_string(20).encode('ascii')).decode('ascii')
+
+ image = 'mcr.microsoft.com/azure-blob-storage'
+ port = 11002
+ environment = {'LOCAL_STORAGE_ACCOUNT_NAME': account,
'LOCAL_STORAGE_ACCOUNT_KEY': secret}
+ ready_message = b'BlobService - StartAsync completed'
+
+
+class StorageTest(Integration.TestBase):
+ provider = 'azure_blobs'
+
+ kind = storage_models.Kind.STORAGE
+ access_tier = None # type: storage_models.AccessTier
+
+ @classmethod
+ def setUpClass(cls):
+ if identity is None:
+ raise unittest.SkipTest('missing azure-identity library')
+
+ if resource is None or resource_models is None:
+ raise unittest.SkipTest('missing azure-mgmt-resource library')
+
+ if storage is None or storage_models is None:
+ raise unittest.SkipTest('missing azure-mgmt-storage library')
+
+ config = {
+ key: os.getenv(key)
+ for key in (
+ 'AZURE_TENANT_ID',
+ 'AZURE_SUBSCRIPTION_ID',
+ 'AZURE_CLIENT_ID',
+ 'AZURE_CLIENT_SECRET',
+ )
+ }
+
+ for key, value in config.items():
+ if not value:
+ raise unittest.SkipTest('missing environment variable %s' %
key)
+
+ credentials = identity.ClientSecretCredential(
+ tenant_id=config['AZURE_TENANT_ID'],
+ client_id=config['AZURE_CLIENT_ID'],
+ client_secret=config['AZURE_CLIENT_SECRET'],
+ )
+
+ resource_client = resource.ResourceManagementClient(
+ credentials,
+ config['AZURE_SUBSCRIPTION_ID'],
+ )
+
+ storage_client = storage.StorageManagementClient(
+ credentials,
+ config['AZURE_SUBSCRIPTION_ID'],
+ )
+
+ location = os.getenv('AZURE_LOCATION', DEFAULT_AZURE_LOCATION)
+ name = 'libcloud'
+ name += random_string(MAX_STORAGE_ACCOUNT_NAME_LENGTH - len(name))
+ timeout = float(os.getenv('AZURE_TIMEOUT_SECONDS',
DEFAULT_TIMEOUT_SECONDS))
+
+ group = resource_client.resource_groups.create_or_update(
+ resource_group_name=name,
+ parameters=resource_models.ResourceGroup(
+ location=location,
+ tags={
+ 'test': cls.__name__,
+ 'run': os.getenv('GITHUB_RUN_ID', '-'),
+ },
+ ),
+ timeout=timeout,
+ )
+
+ cls.addClassCleanup(lambda: resource_client.resource_groups
+ .begin_delete(group.name)
+ .result(timeout))
+
+ account = storage_client.storage_accounts.begin_create(
+ resource_group_name=group.name,
+ account_name=name,
+ parameters=storage_models.StorageAccountCreateParameters(
+
sku=storage_models.Sku(name=storage_models.SkuName.STANDARD_LRS),
+ access_tier=cls.access_tier,
+ kind=cls.kind,
+ location=location,
+ ),
+ ).result(timeout)
+
+ keys = storage_client.storage_accounts.list_keys(
+ resource_group_name=group.name,
+ account_name=account.name,
+ timeout=timeout,
+ )
+
+ cls.account = account.name
+ cls.secret = keys.keys[0].value
+
+
+class StorageV2Test(StorageTest):
+ kind = storage_models.Kind.STORAGE_V2
+
+
+class BlobStorageTest(StorageTest):
+ kind = storage_models.Kind.BLOB_STORAGE
+ access_tier = storage_models.AccessTier.HOT
+
+
+if __name__ == '__main__':
+ sys.exit(unittest.main())
diff --git a/tox.ini b/tox.ini
index e633fc3..7a22ab9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,9 +1,9 @@
[tox]
-envlist = py{pypy3.5,3.5,3.6,3.7,3.8,3.9},checks,lint,pylint,mypy,docs,coverage
+envlist =
py{pypy3.5,3.5,3.6,3.7,3.8,3.9},checks,lint,pylint,mypy,docs,coverage,integration-storage
skipsdist = true
[testenv]
-passenv = TERM CI GITHUB_*
+passenv = TERM CI GITHUB_* DOCKER_*
deps =
-r{toxinidir}/requirements-tests.txt
fasteners
@@ -14,7 +14,7 @@ basepython =
pypypy3: pypy3
py3.5: python3.5
py3.6: python3.6
-
{py3.7,docs,checks,lint,pylint,mypy,coverage,docs,py3.7-dist,py3.7-dist-wheel}:
python3.7
+
{py3.7,docs,checks,lint,pylint,mypy,coverage,docs,integration-storage,py3.7-dist,py3.7-dist-wheel}:
python3.7
{py3.8,py3.8-windows}: python3.8
{py3.9}: python3.9
setenv =
@@ -198,7 +198,7 @@ deps = -r{toxinidir}/requirements-tests.txt
commands = flake8 libcloud/
flake8 --max-line-length=160 libcloud/test/
flake8 demos/
- flake8 integration/
+ flake8 --max-line-length=160 integration/
flake8 scripts/
flake8 --ignore=E402,E902,W503,W504 docs/examples/
flake8 --ignore=E402,E902,W503,W504 --max-line-length=160 contrib/
@@ -212,10 +212,22 @@ commands =
bash ./scripts/check_file_names.sh
python ./scripts/check_asf_license_headers.py .
-[testenv:integration]
-deps = -r{toxinidir}/integration/requirements.txt
+[testenv:integration-compute]
+deps = -r{toxinidir}/integration/compute/requirements.txt
-commands = python -m integration
+commands = python -m integration.compute
+
+[testenv:integration-storage]
+passenv = AZURE_CLIENT_SECRET
+
+setenv =
+ AZURE_CLIENT_ID=16cd65a3-dfa2-4272-bcdb-842cbbedb1b7
+ AZURE_TENANT_ID=982317c6-fb7e-4e92-abcd-196557e41c5b
+ AZURE_SUBSCRIPTION_ID=d6d608a6-e0c8-42ae-a548-2f41793709d2
+
+deps = -r{toxinidir}/integration/storage/requirements.txt
+
+commands = python -m integration.storage
[testenv:coverage]
deps =
