Repository: libcloud Updated Branches: refs/heads/trunk f01509da5 -> 930d05a43
LIBCLOUD-625: Allow for internal GCE authorization with metadata service Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/20d97707 Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/20d97707 Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/20d97707 Branch: refs/heads/trunk Commit: 20d977075117f05a0d8cd8ceb91c4dfcd93a7766 Parents: a38ade5 Author: Eric Johnson <[email protected]> Authored: Sat Oct 18 02:17:00 2014 +0000 Committer: Eric Johnson <[email protected]> Committed: Tue Oct 21 16:10:49 2014 +0000 ---------------------------------------------------------------------- demos/secrets.py-dist | 2 +- docs/compute/drivers/gce.rst | 21 +++- docs/examples/compute/gce/gce_internal_auth.py | 9 ++ libcloud/common/google.py | 108 ++++++++++++++++---- libcloud/compute/drivers/gce.py | 24 +++-- libcloud/test/common/test_google.py | 8 ++ libcloud/test/secrets.py-dist | 2 +- libcloud/utils/connection.py | 7 +- 8 files changed, 148 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/20d97707/demos/secrets.py-dist ---------------------------------------------------------------------- diff --git a/demos/secrets.py-dist b/demos/secrets.py-dist index 82c3de1..981286d 100644 --- a/demos/secrets.py-dist +++ b/demos/secrets.py-dist @@ -22,7 +22,7 @@ DREAMHOST_PARAMS = ('key',) EC2_PARAMS = ('access_id', 'secret') ECP_PARAMS = ('user_name', 'password') GANDI_PARAMS = ('user',) -GCE_PARAMS = ('email_address', 'key') # Service Account Authentication +GCE_PARAMS = ('[email protected]', 'key') # Service Account Authentication #GCE_PARAMS = ('client_id', 'client_secret') # Installed App Authentication GCE_KEYWORD_PARAMS = {'project': 'project_name'} HOSTINGCOM_PARAMS = ('user', 'secret') http://git-wip-us.apache.org/repos/asf/libcloud/blob/20d97707/docs/compute/drivers/gce.rst ---------------------------------------------------------------------- diff --git a/docs/compute/drivers/gce.rst b/docs/compute/drivers/gce.rst index 6e91160..9230ca9 100644 --- a/docs/compute/drivers/gce.rst +++ b/docs/compute/drivers/gce.rst @@ -20,8 +20,8 @@ Google Compute Engine features: Connecting to Google Compute Engine ----------------------------------- -Libcloud supports two different methods for authenticating to Compute Engine: -`Service Account`_ and `Installed Application`_ +Libcloud supports three different methods for authenticating: +`Service Account`_ and `Installed Application`_ and `Internal Authentication_` Which one should I use? @@ -34,6 +34,11 @@ Which one should I use? example, a desktop application for managing VMs that would be used by many different people with different Google accounts. +* If you are running your code on an instance inside Google Compute Engine, + the GCE driver will consult the internal metadata service to obtain an + authorization token. The only parameter required for this type of + authorization is your Project ID. + Once you have set up the authentication as described below, you pass the authentication information to the driver as described in `Examples`_ @@ -69,6 +74,13 @@ To set up Installed Account authentication: 7. You will also need your "Project ID" which can be found by clicking on the "Overview" link on the left sidebar. +Internal Authentication +~~~~~~~~~~~~~~~~~~~~~~~ + +To use GCE's internal metadata service to authenticate, simply specify +your Project ID and let the driver handle the rest. See the `Examples`_ +below for a sample. + Accessing Google Cloud services from your Libcloud nodes -------------------------------------------------------- In order for nodes created with libcloud to be able to access or manage other @@ -103,6 +115,11 @@ https://github.com/apache/libcloud/blob/trunk/demos/gce_demo.py .. literalinclude:: /examples/compute/gce/gce_service_account_scopes.py +5. Using GCE Internal Authorization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. literalinclude:: /examples/compute/gce/gce_internal_auth.py + API Docs -------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/20d97707/docs/examples/compute/gce/gce_internal_auth.py ---------------------------------------------------------------------- diff --git a/docs/examples/compute/gce/gce_internal_auth.py b/docs/examples/compute/gce/gce_internal_auth.py new file mode 100644 index 0000000..d3aa6ac --- /dev/null +++ b/docs/examples/compute/gce/gce_internal_auth.py @@ -0,0 +1,9 @@ +from libcloud.compute.types import Provider +from libcloud.compute.providers import get_driver + +# This example assumes you are running on an instance within Google +# Compute Engine. As such, the only parameter you need to specify is +# the Project ID. The GCE driver will the consult GCE's internal +# metadata service for an authorization token. +ComputeEngine = get_driver(Provider.GCE) +driver = ComputeEngine(project='your_project_id') http://git-wip-us.apache.org/repos/asf/libcloud/blob/20d97707/libcloud/common/google.py ---------------------------------------------------------------------- diff --git a/libcloud/common/google.py b/libcloud/common/google.py index 52692e6..b805ee1 100644 --- a/libcloud/common/google.py +++ b/libcloud/common/google.py @@ -78,6 +78,7 @@ import os import socket import sys +from libcloud.utils.connection import get_response_object from libcloud.utils.py3 import httplib, urlencode, urlparse, PY3 from libcloud.common.base import (ConnectionUserAndKey, JsonResponse, PollingConnection) @@ -99,6 +100,23 @@ except ImportError: TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%SZ' +def _is_gce(): + http_code, http_reason, body = _get_gce_metadata() + if http_code == httplib.OK and body: + return True + return False + + +def _get_gce_metadata(path=''): + try: + url = "http://metadata/computeMetadata/v1/"; + path.lstrip('/') + headers = {'Metadata-Flavor': 'Google'} + response = get_response_object(url, headers) + return response.status, "", response.body + except Exception as e: + return -1, str(e), None + + class GoogleAuthError(LibcloudError): """Generic Error class for various authentication errors.""" def __init__(self, value): @@ -123,7 +141,16 @@ class JsonParseError(GoogleBaseError): class ResourceNotFoundError(GoogleBaseError): - pass + def __init__(self, value, http_code, code, driver=None): + self.code = code + if isinstance(value, dict) and 'message' in value and \ + value['message'].count('/') == 1 and \ + value['message'].count('projects/') == 1: + value['message'] = value['message'] + ". A missing project " \ + "error may be an authentication issue. " \ + "Please ensure your auth credentials match " \ + "your project. " + super(GoogleBaseError, self).__init__(value, http_code, driver) class QuotaExceededError(GoogleBaseError): @@ -255,7 +282,7 @@ class GoogleBaseAuthConnection(ConnectionUserAndKey): host = 'accounts.google.com' auth_path = '/o/oauth2/auth' - def __init__(self, user_id, key, scopes=None, + def __init__(self, user_id, key=None, scopes=None, redirect_uri='urn:ietf:wg:oauth:2.0:oob', login_hint=None, **kwargs): """ @@ -318,6 +345,21 @@ class GoogleBaseAuthConnection(ConnectionUserAndKey): token_info['expire_time'] = expire_time.strftime(TIMESTAMP_FORMAT) return token_info + def refresh_token(self, token_info): + """ + Refresh the current token. + + Fetch an updated refresh token from internal metadata service. + + :param token_info: Dictionary containing token information. + (Not used, but here for compatibility) + :type token_info: ``dict`` + + :return: A dictionary containing updated token information. + :rtype: ``dict`` + """ + return self.get_new_token() + class GoogleInstalledAppAuthConnection(GoogleBaseAuthConnection): """Authentication connection for "Installed Application" authentication.""" @@ -450,21 +492,32 @@ class GoogleServiceAcctAuthConnection(GoogleBaseAuthConnection): return self._token_request(request) - def refresh_token(self, token_info): - """ - Refresh the current token. - - Service Account authentication doesn't supply a "refresh token" so - this simply gets a new token using the email address/key. - :param token_info: Dictionary containing token information. - (Not used, but here for compatibility) - :type token_info: ``dict`` +class GoogleGCEServiceAcctAuthConnection(GoogleBaseAuthConnection): + """Authentication class for self-authentication when used with a GCE + istance that supports serviceAccounts. + """ + def get_new_token(self): + """ + Get a new token from the internal metadata service. - :return: A dictionary containing updated token information. + :return: Dictionary containing token information :rtype: ``dict`` """ - return self.get_new_token() + path = '/instance/service-accounts/default/token' + http_code, http_reason, token_info = _get_gce_metadata(path) + if http_code == httplib.NOT_FOUND: + raise ValueError("Service Accounts are not enabled for this " + "GCE instance.") + if http_code != httplib.OK: + raise ValueError("Internal GCE Authorization failed: " + "'%s'" % str(http_reason)) + token_info = json.loads(token_info) + if 'expires_in' in token_info: + expire_time = self._now() + datetime.timedelta( + seconds=token_info['expires_in']) + token_info['expire_time'] = expire_time.strftime(TIMESTAMP_FORMAT) + return token_info class GoogleBaseConnection(ConnectionUserAndKey, PollingConnection): @@ -475,7 +528,7 @@ class GoogleBaseConnection(ConnectionUserAndKey, PollingConnection): poll_interval = 2.0 timeout = 180 - def __init__(self, user_id, key, auth_type=None, + def __init__(self, user_id, key=None, auth_type=None, credential_file=None, scopes=None, **kwargs): """ Determine authentication type, set up appropriate authentication @@ -490,10 +543,16 @@ class GoogleBaseConnection(ConnectionUserAndKey, PollingConnection): authentication. :type key: ``str`` - :keyword auth_type: Accepted values are "SA" or "IA" - ("Service Account" or "Installed Application"). + :keyword auth_type: Accepted values are "SA" or "IA" or "GCE" + ("Service Account" or "Installed Application" or + "GCE" if libcloud is being used on a GCE instance + with service account enabled). + If not supplied, auth_type will be guessed based + on value of user_id or if the code is being + executed in a GCE instance.). If not supplied, auth_type will be guessed based - on value of user_id. + on value of user_id or if the code is running + on a GCE instance. :type auth_type: ``str`` :keyword credential_file: Path to file for caching authentication @@ -507,10 +566,11 @@ class GoogleBaseConnection(ConnectionUserAndKey, PollingConnection): self.credential_file = credential_file or '~/.gce_libcloud_auth' if auth_type is None: - # Try to guess. Service accounts use an email address - # as the user id. + # Try to guess. if '@' in user_id: auth_type = 'SA' + elif _is_gce(): + auth_type = 'GCE' else: auth_type = 'IA' @@ -525,14 +585,20 @@ class GoogleBaseConnection(ConnectionUserAndKey, PollingConnection): ] self.token_info = self._get_token_info_from_file() - if auth_type == 'SA': + if auth_type == 'GCE': + self.auth_conn = GoogleGCEServiceAcctAuthConnection( + user_id, self.scopes, **kwargs) + elif auth_type == 'SA': + if '@' not in user_id: + raise GoogleAuthError('Service Account auth requires a ' + 'valid email address') self.auth_conn = GoogleServiceAcctAuthConnection( user_id, key, self.scopes, **kwargs) elif auth_type == 'IA': self.auth_conn = GoogleInstalledAppAuthConnection( user_id, key, self.scopes, **kwargs) else: - raise GoogleAuthError('auth_type should be \'SA\' or \'IA\'') + raise GoogleAuthError('Invalid auth_type: %s' % str(auth_type)) if self.token_info is None: self.token_info = self.auth_conn.get_new_token() http://git-wip-us.apache.org/repos/asf/libcloud/blob/20d97707/libcloud/compute/drivers/gce.py ---------------------------------------------------------------------- diff --git a/libcloud/compute/drivers/gce.py b/libcloud/compute/drivers/gce.py index 856c334..648ae3e 100644 --- a/libcloud/compute/drivers/gce.py +++ b/libcloud/compute/drivers/gce.py @@ -550,8 +550,8 @@ class GCENodeDriver(NodeDriver): "userinfo-email": "userinfo.email" } - def __init__(self, user_id, key, datacenter=None, project=None, - auth_type=None, scopes=None, **kwargs): + def __init__(self, user_id, key=None, datacenter=None, project=None, + auth_type=None, scopes=None, credential_file=None, **kwargs): """ :param user_id: The email address (for service accounts) or Client ID (for installed apps) to be used for authentication. @@ -569,19 +569,30 @@ class GCENodeDriver(NodeDriver): :keyword project: Your GCE project name. (required) :type project: ``str`` - :keyword auth_type: Accepted values are "SA" or "IA" - ("Service Account" or "Installed Application"). + :keyword auth_type: Accepted values are "SA" or "IA" or "GCE" + ("Service Account" or "Installed Application" or + "GCE" if libcloud is being used on a GCE instance + with service account enabled). If not supplied, auth_type will be guessed based - on value of user_id. + on value of user_id or if the code is being + executed in a GCE instance. :type auth_type: ``str`` :keyword scopes: List of authorization URLs. Default is empty and grants read/write to Compute, Storage, DNS. :type scopes: ``list`` + + :keyword credential_file: Path to file for caching authentication + information used by GCEConnection. + :type credential_file: ``str`` """ + self.auth_type = auth_type self.project = project self.scopes = scopes + self.credential_file = credential_file or \ + '~/.gce_libcloud_auth' + '.' + self.project + if not self.project: raise ValueError('Project name must be specified using ' '"project" keyword.') @@ -2749,7 +2760,8 @@ class GCENodeDriver(NodeDriver): def _ex_connection_class_kwargs(self): return {'auth_type': self.auth_type, 'project': self.project, - 'scopes': self.scopes} + 'scopes': self.scopes, + 'credential_file': self.credential_file} def _catch_error(self, ignore_errors=False): """ http://git-wip-us.apache.org/repos/asf/libcloud/blob/20d97707/libcloud/test/common/test_google.py ---------------------------------------------------------------------- diff --git a/libcloud/test/common/test_google.py b/libcloud/test/common/test_google.py index 2e9c701..d851b06 100644 --- a/libcloud/test/common/test_google.py +++ b/libcloud/test/common/test_google.py @@ -31,6 +31,7 @@ from libcloud.common.google import (GoogleAuthError, GoogleBaseAuthConnection, GoogleInstalledAppAuthConnection, GoogleServiceAcctAuthConnection, + GoogleGCEServiceAcctAuthConnection, GoogleBaseConnection) from libcloud.test.secrets import GCE_PARAMS @@ -125,6 +126,8 @@ class GoogleBaseConnectionTest(LibcloudTestCase): GoogleInstalledAppAuthConnection.get_code = lambda x: '1234' GoogleServiceAcctAuthConnection.get_new_token = \ lambda x: x._token_request({}) + GoogleGCEServiceAcctAuthConnection.get_new_token = \ + lambda x: x._token_request({}) GoogleBaseConnection._now = lambda x: datetime.datetime(2013, 6, 26, 19, 0, 0) @@ -152,6 +155,11 @@ class GoogleBaseConnectionTest(LibcloudTestCase): self.assertTrue(isinstance(conn2.auth_conn, GoogleInstalledAppAuthConnection)) + kwargs['auth_type'] = 'GCE' + conn3 = GoogleBaseConnection(*GCE_PARAMS, **kwargs) + self.assertTrue(isinstance(conn3.auth_conn, + GoogleGCEServiceAcctAuthConnection)) + def test_add_default_headers(self): old_headers = {} new_expected_headers = {'Content-Type': 'application/json', http://git-wip-us.apache.org/repos/asf/libcloud/blob/20d97707/libcloud/test/secrets.py-dist ---------------------------------------------------------------------- diff --git a/libcloud/test/secrets.py-dist b/libcloud/test/secrets.py-dist index b3525e8..940f455 100644 --- a/libcloud/test/secrets.py-dist +++ b/libcloud/test/secrets.py-dist @@ -22,7 +22,7 @@ DREAMHOST_PARAMS = ('key',) EC2_PARAMS = ('access_id', 'secret') ECP_PARAMS = ('user_name', 'password') GANDI_PARAMS = ('user',) -GCE_PARAMS = ('email_address', 'key') # Service Account Authentication +GCE_PARAMS = ('[email protected]', 'key') # Service Account Authentication # GCE_PARAMS = ('client_id', 'client_secret') # Installed App Authentication GCE_KEYWORD_PARAMS = {'project': 'project_name'} HOSTINGCOM_PARAMS = ('user', 'secret') http://git-wip-us.apache.org/repos/asf/libcloud/blob/20d97707/libcloud/utils/connection.py ---------------------------------------------------------------------- diff --git a/libcloud/utils/connection.py b/libcloud/utils/connection.py index db381a3..f507ad3 100644 --- a/libcloud/utils/connection.py +++ b/libcloud/utils/connection.py @@ -21,7 +21,7 @@ __all__ = [ ] -def get_response_object(url): +def get_response_object(url, headers=None): """ Utility function which uses libcloud's connection class to issue an HTTP request. @@ -29,6 +29,9 @@ def get_response_object(url): :param url: URL to send the request to. :type url: ``str`` + :param headers: Custom request headers. + :type headers: ``dict`` + :return: Response object. :rtype: :class:`Response`. """ @@ -38,5 +41,5 @@ def get_response_object(url): con = Connection(secure=secure, host=parsed_url.netloc) response = con.request(method='GET', action=parsed_url.path, - params=parsed_qs) + params=parsed_qs, headers=headers) return response
