Extended the ECS driver to support the new container registry API, with tests and doc examples
Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/f00a9b75 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/f00a9b75 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/f00a9b75 Branch: refs/heads/trunk Commit: f00a9b7599cb01e5885f160f2e7e31e24c0598be Parents: d29629d Author: anthony-shaw <anthony.p.s...@gmail.com> Authored: Mon Jan 11 15:19:24 2016 +1100 Committer: anthony-shaw <anthony.p.s...@gmail.com> Committed: Mon Jan 11 15:19:24 2016 +1100 ---------------------------------------------------------------------- docs/container/drivers/ecs.rst | 13 ++ .../container/ecs/container_registry.py | 30 +++++ libcloud/common/aws.py | 3 +- libcloud/container/drivers/ecs.py | 123 +++++++++++++++---- libcloud/container/utils/docker.py | 8 +- .../container/fixtures/ecs/createservice.json | 30 +++++ .../container/fixtures/ecs/deleteservice.json | 30 +++++ .../fixtures/ecs/describerepositories.json | 8 ++ .../fixtures/ecs/describeservices.json | 33 +++++ .../fixtures/ecs/getauthorizationtoken.json | 9 ++ .../test/container/fixtures/ecs/listimages.json | 7 ++ .../container/fixtures/ecs/listservices.json | 6 + libcloud/test/container/test_ecs.py | 41 ++++++- 13 files changed, 309 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/f00a9b75/docs/container/drivers/ecs.rst ---------------------------------------------------------------------- diff --git a/docs/container/drivers/ecs.rst b/docs/container/drivers/ecs.rst index fbde93f..4dc3912 100644 --- a/docs/container/drivers/ecs.rst +++ b/docs/container/drivers/ecs.rst @@ -34,6 +34,19 @@ You can use this class for fetching images to deploy to services like ECS .. literalinclude:: /examples/container/docker_hub.py :language: python +Deploying a container from Amazon Elastic Container Registry (ECR) +------------------------------------------------------------------ + +Amazon ECR is a combination of the Docker Registry V2 API and a proprietary API. The ECS driver includes methods for talking to both APIs. + +Docker Registry API Client :class:`~libcloud.container.utils.docker.RegistryClient` is a shared utility class for interfacing to the public Docker Hub Service. + +You can use a factory method to generate an instance of RegsitryClient from the ECS driver. This will request a 12 hour token from the Amazon API and instantiate a :class:`~libcloud.container.utils.docker.RegistryClient` +object with those credentials. + +.. literalinclude:: /examples/container/container_registry.py + :language: python + API Docs -------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/f00a9b75/docs/examples/container/ecs/container_registry.py ---------------------------------------------------------------------- diff --git a/docs/examples/container/ecs/container_registry.py b/docs/examples/container/ecs/container_registry.py new file mode 100644 index 0000000..2003d5b --- /dev/null +++ b/docs/examples/container/ecs/container_registry.py @@ -0,0 +1,30 @@ +from libcloud.container.types import Provider +from libcloud.container.providers import get_driver + +cls = get_driver(Provider.ECS) + +# Connect to AWS +conn = cls(access_id='SDHFISJDIFJSIDFJ', + secret='THIS_IS)+_MY_SECRET_KEY+I6TVkv68o4H', + region='ap-southeast-2') + +# Get a Registry API client for an existing repository +client = conn.ex_get_registry_client('my-image') + +# List all the images +for image in client.list_images('my-image'): + print(image.name) + +# Get a specific image +image = client.get_image('my-image', '14.04') + +print(image.path) +# >> 647433528374.dkr.ecr.region.amazonaws.com/my-images:14.04 + +# Deploy that image +cluster = conn.list_clusters()[0] +container = conn.deploy_container( + cluster=cluster, + name='my-simple-app', + image=image +) http://git-wip-us.apache.org/repos/asf/libcloud/blob/f00a9b75/libcloud/common/aws.py ---------------------------------------------------------------------- diff --git a/libcloud/common/aws.py b/libcloud/common/aws.py index b7153e3..2cc1d4b 100644 --- a/libcloud/common/aws.py +++ b/libcloud/common/aws.py @@ -394,8 +394,7 @@ class AWSJsonResponse(JsonResponse): def parse_error(self): response = json.loads(self.body) code = response['__type'] - message = response['Message'] - + message = response.get('Message', response['message']) return ('%s: %s' % (code, message)) http://git-wip-us.apache.org/repos/asf/libcloud/blob/f00a9b75/libcloud/container/drivers/ecs.py ---------------------------------------------------------------------- diff --git a/libcloud/container/drivers/ecs.py b/libcloud/container/drivers/ecs.py index 7f6faf3..36c5be3 100644 --- a/libcloud/container/drivers/ecs.py +++ b/libcloud/container/drivers/ecs.py @@ -21,6 +21,7 @@ except ImportError: from libcloud.container.base import (ContainerDriver, Container, ContainerCluster, ContainerImage) from libcloud.container.types import ContainerState +from libcloud.container.utils.docker import RegistryClient from libcloud.common.aws import SignedAWSConnection, AWSJsonResponse __all__ = [ @@ -56,6 +57,7 @@ class ECRJsonConnection(SignedAWSConnection): class ElasticContainerDriver(ContainerDriver): name = 'Amazon Elastic Container Service' website = 'https://aws.amazon.com/ecs/details/' + ecr_repository_host = '%s.dkr.ecr.%s.amazonaws.com' connectionCls = ECSJsonConnection ecrConnectionClass = ECRJsonConnection supports_clusters = False @@ -80,7 +82,7 @@ class ElasticContainerDriver(ContainerDriver): def _ex_connection_class_kwargs(self): return {'signature_version': '4'} - def list_images(self, ex_repository_name=None): + def list_images(self, ex_repository_name): """ List the images in an ECR repository @@ -91,14 +93,19 @@ class ElasticContainerDriver(ContainerDriver): :return: a list of images :rtype: ``list`` of :class:`libcloud.container.base.ContainerImage` """ - request = {'repositoryName': 'default'} + request = {} + request['repositoryName'] = ex_repository_name list_response = self.ecr_connection.request( ROOT, method='POST', data=json.dumps(request), headers=self._get_ecr_headers('ListImages') ).object - return self._to_images(list_response) + repository_id = self.ex_get_repository_id(ex_repository_name) + host = self._get_ecr_host(repository_id) + return self._to_images(list_response['imageIds'], + host, + ex_repository_name) def list_clusters(self): """ @@ -415,7 +422,7 @@ class ElasticContainerDriver(ContainerDriver): data=json.dumps(request), headers=self._get_headers('CreateService') ).object - return response + return response['service'] def ex_list_service_arns(self, cluster=None): """ @@ -437,7 +444,7 @@ class ElasticContainerDriver(ContainerDriver): ).object return response['serviceArns'] - def ex_describe_service(self, cluster, service_arn): + def ex_describe_service(self, service_arn): """ Get the details of a service @@ -451,8 +458,6 @@ class ElasticContainerDriver(ContainerDriver): :rtype: ``object`` """ request = {'services': [service_arn]} - if cluster is not None: - request['cluster'] = cluster.id response = self.connection.request( ROOT, method='POST', @@ -461,7 +466,7 @@ class ElasticContainerDriver(ContainerDriver): ).object return response['services'][0] - def ex_destroy_service(self, cluster, service_arn): + def ex_destroy_service(self, service_arn): """ Deletes a service @@ -472,37 +477,96 @@ class ElasticContainerDriver(ContainerDriver): :type service_arn: ``str`` """ request = { - 'service': service_arn, - 'cluster': cluster.id} + 'service': service_arn} response = self.connection.request( ROOT, method='POST', data=json.dumps(request), headers=self._get_headers('DeleteService') ).object - return response + return response['service'] + + def ex_get_registry_client(self, repository_name): + """ + Get a client for an ECR repository + + :param repository_name: The unique name of the repository + :type repository_name: ``str`` + + :return: a docker registry API client + :rtype: :class:`libcloud.container.utils.docker.RegistryClient` + """ + repository_id = self.ex_get_repository_id(repository_name) + token = self.ex_get_repository_token(repository_id) + host = self._get_ecr_host(repository_id) + return RegistryClient( + host=host, + username='AWS', + password=token + ) + + def ex_get_repository_token(self, repository_id): + """ + Get the authorization token (12 hour expiry) for a repository + + :param repository_id: The ID of the repository + :type repository_id: ``str`` + + :return: A token for login + :rtype: ``str`` + """ + request = {'RegistryIds': [repository_id]} + response = self.ecr_connection.request( + ROOT, + method='POST', + data=json.dumps(request), + headers=self._get_ecr_headers('GetAuthorizationToken') + ).object + return response['authorizationData'][0]['authorizationToken'] + + def ex_get_repository_id(self, repository_name): + """ + Get the ID of a repository + + :param repository_name: The unique name of the repository + :type repository_name: ``str`` + + :return: The repository ID + :rtype: ``str`` + """ + request = {'repositoryNames': [repository_name]} + list_response = self.ecr_connection.request( + ROOT, + method='POST', + data=json.dumps(request), + headers=self._get_ecr_headers('DescribeRepositories') + ).object + repository_id = list_response['repositories'][0]['registryId'] + return repository_id + + def _get_ecr_host(self, repository_id): + return self.ecr_repository_host % ( + repository_id, + self.region) def _get_headers(self, action): + """ + Get the default headers for a request to the ECS API + """ return {'x-amz-target': '%s.%s' % (ECS_TARGET_BASE, action), 'Content-Type': 'application/x-amz-json-1.1' } def _get_ecr_headers(self, action): + """ + Get the default headers for a request to the ECR API + """ return {'x-amz-target': '%s.%s' % (ECR_TARGET_BASE, action), 'Content-Type': 'application/x-amz-json-1.1' } - def ex_docker_hub_image(self, name): - return ContainerImage( - id=None, - name=name, - path=None, - version=None, - driver=self.connection.driver - ) - def _to_clusters(self, data): clusters = [] for cluster in data['clusters']: @@ -542,17 +606,22 @@ class ElasticContainerDriver(ContainerDriver): driver=self.connection.driver ) - def _to_images(self, data): + def _to_images(self, data, host, repository_name): images = [] - for image in data['images']: - images.append(self._to_image(image)) + for image in data: + images.append(self._to_image(image, host, repository_name)) return images - def _to_image(self, data): + def _to_image(self, data, host, repository_name): + path = '%s/%s:%s' % ( + host, + repository_name, + data['imageTag'] + ) return ContainerImage( id=None, - name=data['name'], - path=None, - version=None, + name=path, + path=path, + version=data['imageTag'], driver=self.connection.driver ) http://git-wip-us.apache.org/repos/asf/libcloud/blob/f00a9b75/libcloud/container/utils/docker.py ---------------------------------------------------------------------- diff --git a/libcloud/container/utils/docker.py b/libcloud/container/utils/docker.py index 9e9703c..a8a544a 100644 --- a/libcloud/container/utils/docker.py +++ b/libcloud/container/utils/docker.py @@ -63,7 +63,10 @@ class RegistryClient(object): :param password: (optional) Your hub account password :type password: ``str`` """ - self.connection = DockerHubConnection(host, username, password, **kwargs) + self.connection = DockerHubConnection(host, + username, + password, + **kwargs) def list_images(self, repository_name, namespace='library', max_count=100): """ @@ -161,4 +164,5 @@ class HubClient(RegistryClient): :param password: (optional) Your hub account password :type password: ``str`` """ - self.connection = DockerHubConnection(self.host, username, password, **kwargs) + self.connection = DockerHubConnection(self.host, username, + password, **kwargs) http://git-wip-us.apache.org/repos/asf/libcloud/blob/f00a9b75/libcloud/test/container/fixtures/ecs/createservice.json ---------------------------------------------------------------------- diff --git a/libcloud/test/container/fixtures/ecs/createservice.json b/libcloud/test/container/fixtures/ecs/createservice.json new file mode 100644 index 0000000..2f9d421 --- /dev/null +++ b/libcloud/test/container/fixtures/ecs/createservice.json @@ -0,0 +1,30 @@ +{ + "service": { + "clusterArn": "arn:aws:ecs:us-east-1:012345678910:cluster/default", + "deploymentConfiguration": { + "maximumPercent": 200, + "minimumHealthyPercent": 100 + }, + "deployments": [ + { + "createdAt": 1430326887.362, + "desiredCount": 10, + "id": "ecs-svc/9223370606527888445", + "pendingCount": 0, + "runningCount": 0, + "status": "PRIMARY", + "taskDefinition": "arn:aws:ecs:us-east-1:012345678910:task-definition/ecs-demo:1", + "updatedAt": 1430326887.362 + } + ], + "desiredCount": 10, + "events": [], + "loadBalancers": [], + "pendingCount": 0, + "runningCount": 0, + "serviceArn": "arn:aws:ecs:us-east-1:012345678910:service/test", + "serviceName": "test", + "status": "ACTIVE", + "taskDefinition": "arn:aws:ecs:us-east-1:012345678910:task-definition/ecs-demo:1" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/f00a9b75/libcloud/test/container/fixtures/ecs/deleteservice.json ---------------------------------------------------------------------- diff --git a/libcloud/test/container/fixtures/ecs/deleteservice.json b/libcloud/test/container/fixtures/ecs/deleteservice.json new file mode 100644 index 0000000..d89d323 --- /dev/null +++ b/libcloud/test/container/fixtures/ecs/deleteservice.json @@ -0,0 +1,30 @@ +{ + "service": { + "clusterArn": "arn:aws:ecs:us-east-1:012345678910:cluster/default", + "deploymentConfiguration": { + "maximumPercent": 200, + "minimumHealthyPercent": 100 + }, + "deployments": [ + { + "createdAt": 1430320735.285, + "desiredCount": 0, + "id": "ecs-svc/9223370606534040511", + "pendingCount": 0, + "runningCount": 0, + "status": "PRIMARY", + "taskDefinition": "arn:aws:ecs:us-east-1:012345678910:task-definition/sleep360:27", + "updatedAt": 1430320735.285 + } + ], + "desiredCount": 0, + "events": [], + "loadBalancers": [], + "pendingCount": 0, + "runningCount": 0, + "serviceArn": "arn:aws:ecs:us-east-1:012345678910:service/test", + "serviceName": "test", + "status": "DRAINING", + "taskDefinition": "arn:aws:ecs:us-east-1:012345678910:task-definition/sleep360:27" + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/f00a9b75/libcloud/test/container/fixtures/ecs/describerepositories.json ---------------------------------------------------------------------- diff --git a/libcloud/test/container/fixtures/ecs/describerepositories.json b/libcloud/test/container/fixtures/ecs/describerepositories.json new file mode 100644 index 0000000..c794005 --- /dev/null +++ b/libcloud/test/container/fixtures/ecs/describerepositories.json @@ -0,0 +1,8 @@ +{ + "repositories": [{ + "registryId": "647433528374", + "repositoryArn": "arn:aws:ecr:us-east-1:647433528374:repository/my-images", + "repositoryName": "my-images" + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/f00a9b75/libcloud/test/container/fixtures/ecs/describeservices.json ---------------------------------------------------------------------- diff --git a/libcloud/test/container/fixtures/ecs/describeservices.json b/libcloud/test/container/fixtures/ecs/describeservices.json new file mode 100644 index 0000000..76acc75 --- /dev/null +++ b/libcloud/test/container/fixtures/ecs/describeservices.json @@ -0,0 +1,33 @@ +{ + "failures": [], + "services": [ + { + "clusterArn": "arn:aws:ecs:us-west-2:012345678910:cluster/telemetry", + "deploymentConfiguration": { + "maximumPercent": 200, + "minimumHealthyPercent": 100 + }, + "deployments": [ + { + "createdAt": 1432829320.611, + "desiredCount": 4, + "id": "ecs-svc/9223370604025455196", + "pendingCount": 0, + "runningCount": 4, + "status": "PRIMARY", + "taskDefinition": "arn:aws:ecs:us-west-2:012345678910:task-definition/hpcc-t2-medium:1", + "updatedAt": 1432829320.611 + } + ], + "desiredCount": 4, + "events": [], + "loadBalancers": [], + "pendingCount": 0, + "runningCount": 4, + "serviceArn": "arn:aws:ecs:us-west-2:012345678910:service/bunker-buster", + "serviceName": "test", + "status": "ACTIVE", + "taskDefinition": "arn:aws:ecs:us-west-2:012345678910:task-definition/hpcc-t2-medium:1" + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/f00a9b75/libcloud/test/container/fixtures/ecs/getauthorizationtoken.json ---------------------------------------------------------------------- diff --git a/libcloud/test/container/fixtures/ecs/getauthorizationtoken.json b/libcloud/test/container/fixtures/ecs/getauthorizationtoken.json new file mode 100644 index 0000000..469c9ef --- /dev/null +++ b/libcloud/test/container/fixtures/ecs/getauthorizationtoken.json @@ -0,0 +1,9 @@ +{ + "authorizationData": [ + { + "authorizationToken": "QVdTOkNpQzErSHF1ZXZPcUR...", + "expiresAt": 1448878779.809, + "proxyEndpoint": "https://012345678910.dkr.ecr.us-east-1.amazonaws.com"; + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/f00a9b75/libcloud/test/container/fixtures/ecs/listimages.json ---------------------------------------------------------------------- diff --git a/libcloud/test/container/fixtures/ecs/listimages.json b/libcloud/test/container/fixtures/ecs/listimages.json new file mode 100644 index 0000000..ae598f0 --- /dev/null +++ b/libcloud/test/container/fixtures/ecs/listimages.json @@ -0,0 +1,7 @@ +{ + "imageIds": [{ + "imageDigest": "sha256:9bacaf947ed397fcc9afb7359a1a8eaa1f6944ba8cd4ddca1c69bdcf4acf12a2", + "imageTag": "latest" + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/f00a9b75/libcloud/test/container/fixtures/ecs/listservices.json ---------------------------------------------------------------------- diff --git a/libcloud/test/container/fixtures/ecs/listservices.json b/libcloud/test/container/fixtures/ecs/listservices.json new file mode 100644 index 0000000..62a3cf5 --- /dev/null +++ b/libcloud/test/container/fixtures/ecs/listservices.json @@ -0,0 +1,6 @@ +{ + "serviceArns": [ + "arn:aws:ecs:us-east-1:012345678910:service/hello_world", + "arn:aws:ecs:us-east-1:012345678910:service/ecs-simple-service" + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/libcloud/blob/f00a9b75/libcloud/test/container/test_ecs.py ---------------------------------------------------------------------- diff --git a/libcloud/test/container/test_ecs.py b/libcloud/test/container/test_ecs.py index 518daae..73a35b3 100644 --- a/libcloud/test/container/test_ecs.py +++ b/libcloud/test/container/test_ecs.py @@ -19,6 +19,7 @@ from libcloud.test import unittest from libcloud.container.base import ContainerCluster, ContainerImage, Container from libcloud.container.drivers.ecs import ElasticContainerDriver +from libcloud.container.utils.docker import RegistryClient from libcloud.utils.py3 import httplib from libcloud.test.secrets import CONTAINER_PARAMS_ECS @@ -136,6 +137,37 @@ class ElasticContainerDriverTestCase(unittest.TestCase): ) self.assertFalse(container is None) + def test_list_images(self): + images = self.driver.list_images('my-images') + self.assertEqual(len(images), 1) + self.assertEqual(images[0].name, '647433528374.dkr.ecr.region.amazonaws.com/my-images:latest') + + def test_ex_create_service(self): + cluster = self.driver.list_clusters()[0] + task_definition = self.driver.list_containers()[0].extra['taskDefinitionArn'] + service = self.driver.ex_create_service(cluster=cluster, + name='jim', + task_definition=task_definition) + self.assertEqual(service['serviceName'], 'test') + + def test_ex_list_service_arns(self): + arns = self.driver.ex_list_service_arns() + self.assertEqual(len(arns), 2) + + def test_ex_describe_service(self): + arn = self.driver.ex_list_service_arns()[0] + service = self.driver.ex_describe_service(arn) + self.assertEqual(service['serviceName'], 'test') + + def test_ex_destroy_service(self): + arn = self.driver.ex_list_service_arns()[0] + service = self.driver.ex_destroy_service(arn) + self.assertEqual(service['status'], 'DRAINING') + + def test_ex_get_registry_client(self): + client = self.driver.ex_get_registry_client('my-images') + self.assertIsInstance(client, RegistryClient) + class ECSMockHttp(MockHttp): fixtures = ContainerFileFixtures('ecs') @@ -148,7 +180,14 @@ class ECSMockHttp(MockHttp): 'ListClusters': 'listclusters.json', 'RegisterTaskDefinition': 'registertaskdefinition.json', 'RunTask': 'runtask.json', - 'StopTask': 'stoptask.json' + 'StopTask': 'stoptask.json', + 'ListImages': 'listimages.json', + 'DescribeRepositories': 'describerepositories.json', + 'CreateService': 'createservice.json', + 'ListServices': 'listservices.json', + 'DescribeServices': 'describeservices.json', + 'DeleteService': 'deleteservice.json', + 'GetAuthorizationToken': 'getauthorizationtoken.json' } def root(
