Repository: beam Updated Branches: refs/heads/master 60834a903 -> 2c8071e20
[BEAM-1101, BEAM-1068] Remove service account name credential pipeline options Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/549b950b Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/549b950b Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/549b950b Branch: refs/heads/master Commit: 549b950b1b3d4acd07d52297ed9722db3087729c Parents: 60834a9 Author: Sourabh Bajaj <[email protected]> Authored: Thu Apr 13 11:28:47 2017 -0700 Committer: Sourabh Bajaj <[email protected]> Committed: Thu Apr 13 13:21:53 2017 -0700 ---------------------------------------------------------------------- sdks/python/apache_beam/internal/gcp/auth.py | 72 ++++++-------------- .../apache_beam/internal/gcp/auth_test.py | 44 ------------ .../apache_beam/utils/pipeline_options.py | 7 -- 3 files changed, 19 insertions(+), 104 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/549b950b/sdks/python/apache_beam/internal/gcp/auth.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/internal/gcp/auth.py b/sdks/python/apache_beam/internal/gcp/auth.py index 8304658..97e21f4 100644 --- a/sdks/python/apache_beam/internal/gcp/auth.py +++ b/sdks/python/apache_beam/internal/gcp/auth.py @@ -21,7 +21,6 @@ import datetime import json import logging import os -import sys import urllib2 from oauth2client.client import GoogleCredentials @@ -29,8 +28,6 @@ from oauth2client.client import OAuth2Credentials from apache_beam.utils import processes from apache_beam.utils import retry -from apache_beam.utils.pipeline_options import GoogleCloudOptions -from apache_beam.utils.pipeline_options import PipelineOptions # When we are running in GCE, we can authenticate with VM credentials. @@ -135,53 +132,22 @@ def get_service_credentials(): 'https://www.googleapis.com/auth/datastore' ] - # TODO(BEAM-1068): Do not recreate options from sys.argv. - # We are currently being run from the command line. - google_cloud_options = PipelineOptions( - sys.argv).view_as(GoogleCloudOptions) - if google_cloud_options.service_account_name: - if not google_cloud_options.service_account_key_file: - raise AuthenticationException( - 'key file not provided for service account.') - if not os.path.exists(google_cloud_options.service_account_key_file): - raise AuthenticationException( - 'Specified service account key file does not exist.') - - # The following code uses oauth2client >=2.0.0 functionality and if this - # is not available due to import errors will use 1.5.2 functionality. - try: - from oauth2client.service_account import ServiceAccountCredentials - return ServiceAccountCredentials.from_p12_keyfile( - google_cloud_options.service_account_name, - google_cloud_options.service_account_key_file, - private_key_password=None, - scopes=client_scopes) - except ImportError: - with file(google_cloud_options.service_account_key_file) as f: - service_account_key = f.read() - from oauth2client.client import SignedJwtAssertionCredentials - return SignedJwtAssertionCredentials( - google_cloud_options.service_account_name, - service_account_key, - client_scopes, - user_agent=user_agent) - else: - try: - credentials = _GCloudWrapperCredentials(user_agent) - # Check if we are able to get an access token. If not fallback to - # application default credentials. - credentials.get_access_token() - return credentials - except AuthenticationException: - logging.warning('Unable to find credentials from gcloud.') - - # Falling back to application default credentials. - try: - credentials = GoogleCredentials.get_application_default() - credentials = credentials.create_scoped(client_scopes) - logging.debug('Connecting using Google Application Default ' - 'Credentials.') - return credentials - except Exception: - logging.warning('Unable to find default credentials to use.') - raise + try: + credentials = _GCloudWrapperCredentials(user_agent) + # Check if we are able to get an access token. If not fallback to + # application default credentials. + credentials.get_access_token() + return credentials + except AuthenticationException: + logging.warning('Unable to find credentials from gcloud.') + + # Falling back to application default credentials. + try: + credentials = GoogleCredentials.get_application_default() + credentials = credentials.create_scoped(client_scopes) + logging.debug('Connecting using Google Application Default ' + 'Credentials.') + return credentials + except Exception: + logging.warning('Unable to find default credentials to use.') + raise http://git-wip-us.apache.org/repos/asf/beam/blob/549b950b/sdks/python/apache_beam/internal/gcp/auth_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/internal/gcp/auth_test.py b/sdks/python/apache_beam/internal/gcp/auth_test.py deleted file mode 100644 index b1f503e..0000000 --- a/sdks/python/apache_beam/internal/gcp/auth_test.py +++ /dev/null @@ -1,44 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -"""Unit tests for the auth module.""" - -import os -import sys -import unittest - -import mock - -from apache_beam.internal.gcp import auth - - -class AuthTest(unittest.TestCase): - - def test_create_application_client(self): - try: - test_args = [ - 'test', '--service_account_name', 'abc', '--service_account_key_file', - os.path.join(os.path.dirname(__file__), '..', '..', 'tests', - 'data', 'privatekey.p12')] - with mock.patch.object(sys, 'argv', test_args): - credentials = auth.get_service_credentials() - self.assertIsNotNone(credentials) - except NotImplementedError: - self.skipTest('service account tests require pyOpenSSL module.') - - -if __name__ == '__main__': - unittest.main() http://git-wip-us.apache.org/repos/asf/beam/blob/549b950b/sdks/python/apache_beam/utils/pipeline_options.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/utils/pipeline_options.py b/sdks/python/apache_beam/utils/pipeline_options.py index c2a44ad..ecc2e32 100644 --- a/sdks/python/apache_beam/utils/pipeline_options.py +++ b/sdks/python/apache_beam/utils/pipeline_options.py @@ -286,13 +286,6 @@ class GoogleCloudOptions(PipelineOptions): default='us-central1', help='The Google Compute Engine region for creating ' 'Dataflow job.') - parser.add_argument('--service_account_name', - default=None, - help='Name of the service account for Google APIs.') - parser.add_argument('--service_account_key_file', - default=None, - help='Path to a file containing the P12 service ' - 'credentials.') parser.add_argument('--service_account_email', default=None, help='Identity to run virtual machines as.')
