[AIRFLOW-1635] Allow creating GCP connection without requiring a JSON file Closes #2640 from barrywhart/airflow-1635-gcp- json-data-master
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/6dec7acd Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/6dec7acd Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/6dec7acd Branch: refs/heads/v1-9-test Commit: 6dec7acde3b599599cd83cfad3c84fc014b9931e Parents: 50e31f0 Author: Barry Hart <[email protected]> Authored: Thu Sep 28 09:19:43 2017 -0700 Committer: Chris Riccomini <[email protected]> Committed: Tue Oct 3 22:30:03 2017 -0700 ---------------------------------------------------------------------- airflow/contrib/hooks/gcp_api_base_hook.py | 27 +++++++++++++++++++++++-- airflow/www/views.py | 2 ++ 2 files changed, 27 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/6dec7acd/airflow/contrib/hooks/gcp_api_base_hook.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/hooks/gcp_api_base_hook.py b/airflow/contrib/hooks/gcp_api_base_hook.py index 28721d3..e6ca240 100644 --- a/airflow/contrib/hooks/gcp_api_base_hook.py +++ b/airflow/contrib/hooks/gcp_api_base_hook.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import json + import httplib2 from oauth2client.client import GoogleCredentials from oauth2client.service_account import ServiceAccountCredentials @@ -59,20 +61,23 @@ class GoogleCloudBaseHook(BaseHook, LoggingMixin): Returns the Credentials object for Google API """ key_path = self._get_field('key_path', False) + keyfile_dict = self._get_field('keyfile_dict', False) scope = self._get_field('scope', False) kwargs = {} if self.delegate_to: kwargs['sub'] = self.delegate_to - if not key_path: + if not key_path and not keyfile_dict: self.log.info('Getting connection using `gcloud auth` user, since no key file ' 'is defined for hook.') credentials = GoogleCredentials.get_application_default() - else: + elif key_path: if not scope: raise AirflowException('Scope should be defined when using a key file.') scopes = [s.strip() for s in scope.split(',')] + + # Get credentials from a JSON file. if key_path.endswith('.json'): self.log.info('Getting connection using a JSON key file.') credentials = ServiceAccountCredentials\ @@ -82,6 +87,24 @@ class GoogleCloudBaseHook(BaseHook, LoggingMixin): 'use a JSON key file.') else: raise AirflowException('Unrecognised extension for key file.') + else: + if not scope: + raise AirflowException('Scope should be defined when using key JSON.') + scopes = [s.strip() for s in scope.split(',')] + + # Get credentials from JSON data provided in the UI. + try: + keyfile_dict = json.loads(keyfile_dict) + + # Depending on how the JSON was formatted, it may contain + # escaped newlines. Convert those to actual newlines. + keyfile_dict['private_key'] = keyfile_dict['private_key'].replace( + '\\n', '\n') + + credentials = ServiceAccountCredentials\ + .from_json_keyfile_dict(keyfile_dict, scopes) + except json.decoder.JSONDecodeError: + raise AirflowException('Invalid key JSON.') return credentials def _get_access_token(self): http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/6dec7acd/airflow/www/views.py ---------------------------------------------------------------------- diff --git a/airflow/www/views.py b/airflow/www/views.py index a0a0999..ad27238 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -2582,6 +2582,7 @@ class ConnectionModelView(wwwutils.SuperUserMixin, AirflowModelView): 'extra__jdbc__drv_clsname', 'extra__google_cloud_platform__project', 'extra__google_cloud_platform__key_path', + 'extra__google_cloud_platform__keyfile_dict', 'extra__google_cloud_platform__scope', ) verbose_name = "Connection" @@ -2603,6 +2604,7 @@ class ConnectionModelView(wwwutils.SuperUserMixin, AirflowModelView): 'extra__jdbc__drv_clsname': StringField('Driver Class'), 'extra__google_cloud_platform__project': StringField('Project Id'), 'extra__google_cloud_platform__key_path': StringField('Keyfile Path'), + 'extra__google_cloud_platform__keyfile_dict': PasswordField('Keyfile JSON'), 'extra__google_cloud_platform__scope': StringField('Scopes (comma seperated)'), }
