Repository: libcloud Updated Branches: refs/heads/trunk 6bab03553 -> ab7d07bd4
[google compute] Add support for JSON private key format Closes #438 Closes LIBCLOUD-627 Closes LIBCLOUD-657 Signed-off-by: Eric Johnson <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/libcloud/repo Commit: http://git-wip-us.apache.org/repos/asf/libcloud/commit/ab7d07bd Tree: http://git-wip-us.apache.org/repos/asf/libcloud/tree/ab7d07bd Diff: http://git-wip-us.apache.org/repos/asf/libcloud/diff/ab7d07bd Branch: refs/heads/trunk Commit: ab7d07bd4e7197c275a926de8d9639de9bc234a6 Parents: 6bab035 Author: Eric Johnson <[email protected]> Authored: Fri Jan 23 22:32:26 2015 +0000 Committer: Eric Johnson <[email protected]> Committed: Mon Jan 26 15:41:20 2015 +0000 ---------------------------------------------------------------------- CHANGES.rst | 5 +++ docs/compute/drivers/gce.rst | 36 ++++++++++------- docs/examples/compute/gce/gce_internal_auth.py | 7 +++- .../examples/compute/gce/gce_service_account.py | 2 + libcloud/common/google.py | 41 +++++++++++++------- libcloud/test/common/fixtures/google/pkey.json | 7 ++++ libcloud/test/common/fixtures/google/pkey.pem | 15 +++++++ libcloud/test/common/test_google.py | 32 +++++++++++++-- libcloud/test/compute/test_gce.py | 2 +- 9 files changed, 114 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/libcloud/blob/ab7d07bd/CHANGES.rst ---------------------------------------------------------------------- diff --git a/CHANGES.rst b/CHANGES.rst index e7f65c4..ae32eae 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -16,6 +16,11 @@ General Compute ~~~~~~~ +- GCE driver update to support JSON format Service Account files and a PY3 + fix from Siim Põder for LIBCLOUD-627. + (LIBCLOUD-627, LIBCLOUD-657, GITHUB-438) + [Eric Johnson] + - GCE driver fixed for missing param on ex_add_access_config (GITHUB-435) [Peter Mooshammer] http://git-wip-us.apache.org/repos/asf/libcloud/blob/ab7d07bd/docs/compute/drivers/gce.rst ---------------------------------------------------------------------- diff --git a/docs/compute/drivers/gce.rst b/docs/compute/drivers/gce.rst index 0d4e283..31aee82 100644 --- a/docs/compute/drivers/gce.rst +++ b/docs/compute/drivers/gce.rst @@ -37,7 +37,7 @@ Which one should I use? * If you are running your code on an instance inside Google Compute Engine, the GCE driver will consult the internal metadata service to obtain an - authorization token. The only parameter required for this type of + authorization token. The only value required for this type of authorization is your Project ID. Once you have set up the authentication as described below, you pass the @@ -47,19 +47,29 @@ authentication information to the driver as described in `Examples`_ Service Account ~~~~~~~~~~~~~~~ -To set up Service Account authentication: +To set up Service Account authentication, you will need to download the +corresponding private key file in either the new JSON (preferred) format, or +the legacy P12 format. 1. Follow the instructions at https://developers.google.com/console/help/new/#serviceaccounts - to create and download a PKCS-12 private key. -2. Convert the PKCS-12 private key to a .pem file using the following: - ``openssl pkcs12 -in YOURPRIVKEY.p12 -nodes -nocerts - | openssl rsa -out PRIV.pem`` -3. Move the .pem file to a safe location -4. You will need the Service Account's "Email Address" and the path to the - .pem file for authentication. -5. You will also need your "Project ID" which can be found by clicking on the - "Overview" link on the left sidebar. + to create and download the private key. + + a. If you opt for the new preferred JSON format, download the file and + save it to a secure location. + + b. If you opt to use the legacy P12 format: + + Convert the private key to a .pem file using the following: + ``openssl pkcs12 -in YOURPRIVKEY.p12 -nodes -nocerts + | openssl rsa -out PRIV.pem`` + + Move the .pem file to a safe location + +2. You will need the Service Account's "Email Address" and the path to the + key file for authentication. +3. You will also need your "Project ID" (a string, not a numerical value) that + can be found by clicking on the "Overview" link on the left sidebar. Installed Application ~~~~~~~~~~~~~~~~~~~~~ @@ -72,8 +82,8 @@ To set up Installed Account authentication: 4. Click on "Credentials" then "Create New Client ID" 5. Select "Installed application" and "Other" then click "Create Client ID" 6. For authentication, you will need the "Client ID" and the "Client Secret" -7. You will also need your "Project ID" which can be found by clicking on the - "Overview" link on the left sidebar. +7. You will also need your "Project ID" (a string, not a numerical value) that + can be found by clicking on the "Overview" link on the left sidebar. Internal Authentication ~~~~~~~~~~~~~~~~~~~~~~~ http://git-wip-us.apache.org/repos/asf/libcloud/blob/ab7d07bd/docs/examples/compute/gce/gce_internal_auth.py ---------------------------------------------------------------------- diff --git a/docs/examples/compute/gce/gce_internal_auth.py b/docs/examples/compute/gce/gce_internal_auth.py index d3aa6ac..4118a74 100644 --- a/docs/examples/compute/gce/gce_internal_auth.py +++ b/docs/examples/compute/gce/gce_internal_auth.py @@ -2,8 +2,11 @@ from libcloud.compute.types import Provider from libcloud.compute.providers import get_driver # This example assumes you are running on an instance within Google -# Compute Engine. As such, the only parameter you need to specify is +# Compute Engine. As such, the only value you need to specify is # the Project ID. The GCE driver will the consult GCE's internal # metadata service for an authorization token. +# +# You must still place placeholder empty strings for user_id / key +# due to the nature of the driver's __init__() params. ComputeEngine = get_driver(Provider.GCE) -driver = ComputeEngine(project='your_project_id') +driver = ComputeEngine('', '', project='your_project_id') http://git-wip-us.apache.org/repos/asf/libcloud/blob/ab7d07bd/docs/examples/compute/gce/gce_service_account.py ---------------------------------------------------------------------- diff --git a/docs/examples/compute/gce/gce_service_account.py b/docs/examples/compute/gce/gce_service_account.py index cbc265f..edc9415 100644 --- a/docs/examples/compute/gce/gce_service_account.py +++ b/docs/examples/compute/gce/gce_service_account.py @@ -2,5 +2,7 @@ from libcloud.compute.types import Provider from libcloud.compute.providers import get_driver ComputeEngine = get_driver(Provider.GCE) +# Note that the 'PEM file' argument can either be the JSON format or +# the P12 format. driver = ComputeEngine('your_service_account_email', 'path_to_pem_file', project='your_project_id') http://git-wip-us.apache.org/repos/asf/libcloud/blob/ab7d07bd/libcloud/common/google.py ---------------------------------------------------------------------- diff --git a/libcloud/common/google.py b/libcloud/common/google.py index 7e3962a..694cf93 100644 --- a/libcloud/common/google.py +++ b/libcloud/common/google.py @@ -79,7 +79,7 @@ import socket import sys from libcloud.utils.connection import get_response_object -from libcloud.utils.py3 import httplib, urlencode, urlparse, PY3 +from libcloud.utils.py3 import b, httplib, urlencode, urlparse, PY3 from libcloud.common.base import (ConnectionUserAndKey, JsonResponse, PollingConnection) from libcloud.common.types import (ProviderError, @@ -345,7 +345,12 @@ class GoogleBaseAuthConnection(ConnectionUserAndKey): """ data = urlencode(request_body) now = self._now() - response = self.request('/o/oauth2/token', method='POST', data=data) + try: + response = self.request('/o/oauth2/token', method='POST', + data=data) + except AttributeError: + raise GoogleAuthError('Invalid authorization response, please ' + 'check your credentials.') token_info = response.object if 'expires_in' in token_info: expire_time = now + datetime.timedelta( @@ -390,12 +395,12 @@ class GoogleInstalledAppAuthConnection(GoogleBaseAuthConnection): data = urlencode(auth_params) url = 'https://%s%s?%s' % (self.host, self.auth_path, data) - print('Please Go to the following URL and sign in:') + print('\nPlease Go to the following URL and sign in:') print(url) if PY3: - code = input('Enter Code:') + code = input('Enter Code: ') else: - code = raw_input('Enter Code:') + code = raw_input('Enter Code: ') return code def get_new_token(self): @@ -458,11 +463,21 @@ class GoogleServiceAcctAuthConnection(GoogleBaseAuthConnection): raise GoogleAuthError('PyCrypto library required for ' 'Service Account Authentication.') # Check to see if 'key' is a file and read the file if it is. - keypath = os.path.expanduser(key) - is_file_path = os.path.exists(keypath) and os.path.isfile(keypath) - if is_file_path: + if key.find("PRIVATE KEY---") == -1: + # key is a file + keypath = os.path.expanduser(key) + is_file_path = os.path.exists(keypath) and os.path.isfile(keypath) + if not is_file_path: + raise ValueError("Missing (or not readable) key " + "file: '%s'" % key) with open(keypath, 'r') as f: - key = f.read() + contents = f.read() + try: + key = json.loads(contents) + key = key['private_key'] + except ValueError: + key = contents + super(GoogleServiceAcctAuthConnection, self).__init__( user_id, key, *args, **kwargs) @@ -475,7 +490,7 @@ class GoogleServiceAcctAuthConnection(GoogleBaseAuthConnection): """ # The header is always the same header = {'alg': 'RS256', 'typ': 'JWT'} - header_enc = base64.urlsafe_b64encode(json.dumps(header)) + header_enc = base64.urlsafe_b64encode(b(json.dumps(header))) # Construct a claim set claim_set = {'iss': self.user_id, @@ -483,10 +498,10 @@ class GoogleServiceAcctAuthConnection(GoogleBaseAuthConnection): 'aud': 'https://accounts.google.com/o/oauth2/token', 'exp': int(time.time()) + 3600, 'iat': int(time.time())} - claim_set_enc = base64.urlsafe_b64encode(json.dumps(claim_set)) + claim_set_enc = base64.urlsafe_b64encode(b(json.dumps(claim_set))) # The message contains both the header and claim set - message = '%s.%s' % (header_enc, claim_set_enc) + message = b'.'.join((header_enc, claim_set_enc)) # Then the message is signed using the key supplied key = RSA.importKey(self.key) hash_func = SHA256.new(message) @@ -494,7 +509,7 @@ class GoogleServiceAcctAuthConnection(GoogleBaseAuthConnection): signature = base64.urlsafe_b64encode(signer.sign(hash_func)) # Finally the message and signature are sent to get a token - jwt = '%s.%s' % (message, signature) + jwt = b'.'.join((message, signature)) request = {'grant_type': 'urn:ietf:params:oauth:grant-type:jwt-bearer', 'assertion': jwt} http://git-wip-us.apache.org/repos/asf/libcloud/blob/ab7d07bd/libcloud/test/common/fixtures/google/pkey.json ---------------------------------------------------------------------- diff --git a/libcloud/test/common/fixtures/google/pkey.json b/libcloud/test/common/fixtures/google/pkey.json new file mode 100644 index 0000000..b3338ca --- /dev/null +++ b/libcloud/test/common/fixtures/google/pkey.json @@ -0,0 +1,7 @@ +{ + "private_key_id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "private_key": "-----BEGIN PRIVATE KEY-----\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\nxxxxxxxxxxxxxxxxxxxxx\n-----END PRIVATE KEY-----\n", + "client_email": "[email protected]", + "client_id": "foo.apps.googleusercontent.com", + "type": "service_account" +} http://git-wip-us.apache.org/repos/asf/libcloud/blob/ab7d07bd/libcloud/test/common/fixtures/google/pkey.pem ---------------------------------------------------------------------- diff --git a/libcloud/test/common/fixtures/google/pkey.pem b/libcloud/test/common/fixtures/google/pkey.pem new file mode 100644 index 0000000..6e42486 --- /dev/null +++ b/libcloud/test/common/fixtures/google/pkey.pem @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +-----END RSA PRIVATE KEY----- http://git-wip-us.apache.org/repos/asf/libcloud/blob/ab7d07bd/libcloud/test/common/test_google.py ---------------------------------------------------------------------- diff --git a/libcloud/test/common/test_google.py b/libcloud/test/common/test_google.py index d851b06..1f1f2ef 100644 --- a/libcloud/test/common/test_google.py +++ b/libcloud/test/common/test_google.py @@ -18,6 +18,7 @@ Tests for Google Connection classes. import datetime import sys import unittest +import os try: import simplejson as json @@ -33,7 +34,7 @@ from libcloud.common.google import (GoogleAuthError, GoogleServiceAcctAuthConnection, GoogleGCEServiceAcctAuthConnection, GoogleBaseConnection) -from libcloud.test.secrets import GCE_PARAMS + # Skip some tests if PyCrypto is unavailable try: @@ -42,6 +43,21 @@ except ImportError: SHA256 = None +SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) +PEM_KEY = os.path.join(SCRIPT_PATH, "fixtures", "google", "pkey.pem") +JSON_KEY = os.path.join(SCRIPT_PATH, "fixtures", "google", "pkey.json") +with open(JSON_KEY, 'r') as f: + KEY_STR = json.loads(f.read())['private_key'] + + +GCE_PARAMS = ('[email protected]', 'key') +GCE_PARAMS_PEM_KEY = ('[email protected]', PEM_KEY) +GCE_PARAMS_JSON_KEY = ('[email protected]', JSON_KEY) +GCE_PARAMS_KEY = ('[email protected]', KEY_STR) +GCE_PARAMS_IA = ('client_id', 'client_secret') +GCE_PARAMS_GCE = ('foo', 'bar') + + class MockJsonResponse(object): def __init__(self, body): self.object = body @@ -146,17 +162,25 @@ class GoogleBaseConnectionTest(LibcloudTestCase): if SHA256: kwargs['auth_type'] = 'SA' - conn1 = GoogleBaseConnection(*GCE_PARAMS, **kwargs) + conn1 = GoogleBaseConnection(*GCE_PARAMS_PEM_KEY, **kwargs) + self.assertTrue(isinstance(conn1.auth_conn, + GoogleServiceAcctAuthConnection)) + + conn1 = GoogleBaseConnection(*GCE_PARAMS_JSON_KEY, **kwargs) + self.assertTrue(isinstance(conn1.auth_conn, + GoogleServiceAcctAuthConnection)) + + conn1 = GoogleBaseConnection(*GCE_PARAMS_KEY, **kwargs) self.assertTrue(isinstance(conn1.auth_conn, GoogleServiceAcctAuthConnection)) kwargs['auth_type'] = 'IA' - conn2 = GoogleBaseConnection(*GCE_PARAMS, **kwargs) + conn2 = GoogleBaseConnection(*GCE_PARAMS_IA, **kwargs) self.assertTrue(isinstance(conn2.auth_conn, GoogleInstalledAppAuthConnection)) kwargs['auth_type'] = 'GCE' - conn3 = GoogleBaseConnection(*GCE_PARAMS, **kwargs) + conn3 = GoogleBaseConnection(*GCE_PARAMS_GCE, **kwargs) self.assertTrue(isinstance(conn3.auth_conn, GoogleGCEServiceAcctAuthConnection)) http://git-wip-us.apache.org/repos/asf/libcloud/blob/ab7d07bd/libcloud/test/compute/test_gce.py ---------------------------------------------------------------------- diff --git a/libcloud/test/compute/test_gce.py b/libcloud/test/compute/test_gce.py index e726054..c56a746 100644 --- a/libcloud/test/compute/test_gce.py +++ b/libcloud/test/compute/test_gce.py @@ -85,7 +85,7 @@ class GCENodeDriverTest(LibcloudTestCase, TestCaseMixin): obj = self.driver._get_object_by_kind( 'https://www.googleapis.com/compute/v1/projects/project_name/' 'global/targetHttpProxies/web-proxy') - self.assertEquals(obj.name, 'web-proxy') + self.assertEqual(obj.name, 'web-proxy') def test_get_region_from_zone(self): zone1 = self.driver.ex_get_zone('us-central1-a')
