This is an automated email from the ASF dual-hosted git repository.
dstandish pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ef40148170 Add credential configuration file support to Google Cloud
Hook (#31548)
ef40148170 is described below
commit ef40148170ce1110a4c9a859207d40dcc6ad1cd5
Author: Philippe Gagnon <[email protected]>
AuthorDate: Fri May 26 14:47:00 2023 -0400
Add credential configuration file support to Google Cloud Hook (#31548)
Add support to authenticate to GCP using a credential configuration file
explicitly defined in a connection. This allows Airflow users to authenticate
to GCP using external accounts without relying on the ADC mechanism, allowing
the configuration of multiple connections utilizing this mechanism, which
offers more flexibility than service account keys.
---
.../google/cloud/utils/credentials_provider.py | 31 +++++++++++++++-
.../providers/google/common/hooks/base_google.py | 7 ++++
.../connections/gcp.rst | 10 +++++
.../cloud/utils/test_credentials_provider.py | 43 +++++++++++++++++++++-
.../google/common/hooks/test_base_google.py | 6 +++
5 files changed, 95 insertions(+), 2 deletions(-)
diff --git a/airflow/providers/google/cloud/utils/credentials_provider.py
b/airflow/providers/google/cloud/utils/credentials_provider.py
index f10dd3d494..f147d8fff9 100644
--- a/airflow/providers/google/cloud/utils/credentials_provider.py
+++ b/airflow/providers/google/cloud/utils/credentials_provider.py
@@ -23,6 +23,7 @@ from __future__ import annotations
import json
import logging
+import os.path
import tempfile
from contextlib import ExitStack, contextmanager
from typing import Collection, Generator, Sequence
@@ -196,6 +197,7 @@ class _CredentialProvider(LoggingMixin):
self,
key_path: str | None = None,
keyfile_dict: dict[str, str] | None = None,
+ credential_config_file: dict[str, str] | str | None = None,
key_secret_name: str | None = None,
key_secret_project_id: str | None = None,
scopes: Collection[str] | None = None,
@@ -213,6 +215,7 @@ class _CredentialProvider(LoggingMixin):
)
self.key_path = key_path
self.keyfile_dict = keyfile_dict
+ self.credential_config_file = credential_config_file
self.key_secret_name = key_secret_name
self.key_secret_project_id = key_secret_project_id
self.scopes = scopes
@@ -233,6 +236,8 @@ class _CredentialProvider(LoggingMixin):
credentials, project_id =
self._get_credentials_using_key_secret_name()
elif self.keyfile_dict:
credentials, project_id =
self._get_credentials_using_keyfile_dict()
+ elif self.credential_config_file:
+ credentials, project_id =
self._get_credentials_using_credential_config_file()
else:
credentials, project_id = self._get_credentials_using_adc()
@@ -311,9 +316,33 @@ class _CredentialProvider(LoggingMixin):
project_id = credentials.project_id
return credentials, project_id
+ def _get_credentials_using_credential_config_file(self):
+ if isinstance(self.credential_config_file, str) and
os.path.exists(self.credential_config_file):
+ self._log_info(
+ f"Getting connection using credential configuration file:
`{self.credential_config_file}`"
+ )
+ credentials, project_id = google.auth.load_credentials_from_file(
+ self.credential_config_file, scopes=self.scopes
+ )
+ else:
+ with tempfile.NamedTemporaryFile(mode="w+t") as
temp_credentials_fd:
+ if isinstance(self.credential_config_file, dict):
+ self._log_info("Getting connection using credential
configuration dict.")
+
temp_credentials_fd.write(json.dumps(self.credential_config_file))
+ elif isinstance(self.credential_config_file, str):
+ self._log_info("Getting connection using credential
configuration string.")
+ temp_credentials_fd.write(self.credential_config_file)
+
+ temp_credentials_fd.flush()
+ credentials, project_id =
google.auth.load_credentials_from_file(
+ temp_credentials_fd.name, scopes=self.scopes
+ )
+
+ return credentials, project_id
+
def _get_credentials_using_adc(self):
self._log_info(
- "Getting connection using `google.auth.default()` since no key
file is defined for hook."
+ "Getting connection using `google.auth.default()` since no
explicit credentials are provided."
)
credentials, project_id = google.auth.default(scopes=self.scopes)
return credentials, project_id
diff --git a/airflow/providers/google/common/hooks/base_google.py
b/airflow/providers/google/common/hooks/base_google.py
index b80a2d9da3..e68cb074e6 100644
--- a/airflow/providers/google/common/hooks/base_google.py
+++ b/airflow/providers/google/common/hooks/base_google.py
@@ -198,6 +198,9 @@ class GoogleBaseHook(BaseHook):
"project": StringField(lazy_gettext("Project Id"),
widget=BS3TextFieldWidget()),
"key_path": StringField(lazy_gettext("Keyfile Path"),
widget=BS3TextFieldWidget()),
"keyfile_dict": PasswordField(lazy_gettext("Keyfile JSON"),
widget=BS3PasswordFieldWidget()),
+ "credential_config_file": StringField(
+ lazy_gettext("Credential Configuration File"),
widget=BS3TextFieldWidget()
+ ),
"scope": StringField(lazy_gettext("Scopes (comma separated)"),
widget=BS3TextFieldWidget()),
"key_secret_name": StringField(
lazy_gettext("Keyfile Secret Name (in GCP Secret Manager)"),
widget=BS3TextFieldWidget()
@@ -251,14 +254,18 @@ class GoogleBaseHook(BaseHook):
keyfile_dict_json = json.loads(keyfile_dict)
except json.decoder.JSONDecodeError:
raise AirflowException("Invalid key JSON.")
+
key_secret_name: str | None = self._get_field("key_secret_name", None)
key_secret_project_id: str | None =
self._get_field("key_secret_project_id", None)
+ credential_config_file: str | None =
self._get_field("credential_config_file", None)
+
target_principal, delegates =
_get_target_principal_and_delegates(self.impersonation_chain)
credentials, project_id = get_credentials_and_project_id(
key_path=key_path,
keyfile_dict=keyfile_dict_json,
+ credential_config_file=credential_config_file,
key_secret_name=key_secret_name,
key_secret_project_id=key_secret_project_id,
scopes=self.scopes,
diff --git a/docs/apache-airflow-providers-google/connections/gcp.rst
b/docs/apache-airflow-providers-google/connections/gcp.rst
index 5c9519df3e..b2f7243fe7 100644
--- a/docs/apache-airflow-providers-google/connections/gcp.rst
+++ b/docs/apache-airflow-providers-google/connections/gcp.rst
@@ -36,6 +36,11 @@ There are two ways to connect to Google Cloud using Airflow.
Key can be specified as a path to the key file (``Keyfile Path``), as a key
payload (``Keyfile JSON``)
or as secret in Secret Manager (``Keyfile secret name``). Only one way of
defining the key can be used at a time.
If you need to manage multiple keys then you should configure multiple
connections.
+3. Using a `credential configuration file
<https://googleapis.dev/python/google-auth/2.9.0/user-guide.html#external-credentials-workload-identity-federation>`_,
+ by specifying the path to or the content of a valid credential
configuration file.
+ A credential configuration file is a configuration file that typically
contains non-sensitive metadata to instruct
+ the ``google-auth`` library on how to retrieve external subject tokens and
exchange them for service account access
+ tokens.
.. warning:: Additional permissions might be needed
@@ -98,6 +103,11 @@ Keyfile JSON
Not required if using application default credentials.
+Credential Configuration File
+ Credential configuration file JSON or path to a credential configuration
file on the filesystem.
+
+ Not required if using application default credentials.
+
Secret name which holds Keyfile JSON
Name of the secret in Secret Manager which contains a `service account
<https://cloud.google.com/docs/authentication/#service_accounts>`_ key.
diff --git a/tests/providers/google/cloud/utils/test_credentials_provider.py
b/tests/providers/google/cloud/utils/test_credentials_provider.py
index fe792ec290..21976c6f6d 100644
--- a/tests/providers/google/cloud/utils/test_credentials_provider.py
+++ b/tests/providers/google/cloud/utils/test_credentials_provider.py
@@ -21,12 +21,14 @@ import logging
import os
import re
from io import StringIO
+from tempfile import NamedTemporaryFile
from unittest import mock
from unittest.mock import ANY
from uuid import uuid4
import pytest
from google.auth.environment_vars import CREDENTIALS
+from google.auth.exceptions import DefaultCredentialsError
from airflow.exceptions import AirflowException
from airflow.providers.google.cloud.utils.credentials_provider import (
@@ -141,7 +143,7 @@ class TestGetGcpCredentialsAndProjectId:
mock_auth_default.assert_called_once_with(scopes=None)
assert ("CREDENTIALS", "PROJECT_ID") == result
assert (
- "Getting connection using `google.auth.default()` since no key
file is defined for hook."
+ "Getting connection using `google.auth.default()` since no
explicit credentials are provided."
) in caplog.messages
@mock.patch("google.auth.default")
@@ -268,6 +270,45 @@ class TestGetGcpCredentialsAndProjectId:
assert (mock_from_service_account_info.return_value,
self.test_project_id) == result
assert "Getting connection using JSON Dict" in caplog.messages
+ @mock.patch("google.auth.load_credentials_from_file",
return_value=("CREDENTIALS", "PROJECT_ID"))
+ def test_get_credentials_using_credential_config_file(self,
mock_load_credentials_from_file, caplog):
+ with caplog.at_level(
+ level=logging.DEBUG, logger=CRED_PROVIDER_LOGGER_NAME
+ ), NamedTemporaryFile() as temp_file:
+ caplog.clear()
+ result =
get_credentials_and_project_id(credential_config_file=temp_file.name)
+
mock_load_credentials_from_file.assert_called_once_with(temp_file.name,
scopes=None)
+ assert mock_load_credentials_from_file.return_value == result
+ assert (
+ f"Getting connection using credential configuration file:
`{temp_file.name}`" in caplog.messages
+ )
+
+ @mock.patch("google.auth.load_credentials_from_file",
return_value=("CREDENTIALS", "PROJECT_ID"))
+ def test_get_credentials_using_credential_config_dict(self,
mock_load_credentials_from_file, caplog):
+ with caplog.at_level(level=logging.DEBUG,
logger=CRED_PROVIDER_LOGGER_NAME):
+ caplog.clear()
+ result =
get_credentials_and_project_id(credential_config_file={"type":
"external_account"})
+ mock_load_credentials_from_file.assert_called_once()
+ assert mock_load_credentials_from_file.return_value == result
+ assert "Getting connection using credential configuration dict." in
caplog.messages
+
+ @mock.patch("google.auth.load_credentials_from_file",
return_value=("CREDENTIALS", "PROJECT_ID"))
+ def test_get_credentials_using_credential_config_string(self,
mock_load_credentials_from_file, caplog):
+ with caplog.at_level(level=logging.DEBUG,
logger=CRED_PROVIDER_LOGGER_NAME):
+ caplog.clear()
+ result =
get_credentials_and_project_id(credential_config_file='{"type":
"external_account"}')
+ mock_load_credentials_from_file.assert_called_once()
+ assert mock_load_credentials_from_file.return_value == result
+ assert "Getting connection using credential configuration string." in
caplog.messages
+
+ def test_get_credentials_using_credential_config_invalid_string(self,
caplog):
+ with pytest.raises(DefaultCredentialsError), caplog.at_level(
+ level=logging.DEBUG, logger=CRED_PROVIDER_LOGGER_NAME
+ ):
+ caplog.clear()
+ get_credentials_and_project_id(credential_config_file="invalid
json}}}}")
+ assert "Getting connection using credential configuration string." in
caplog.messages
+
@mock.patch("google.auth.default", return_value=("CREDENTIALS",
"PROJECT_ID"))
@mock.patch("google.oauth2.service_account.Credentials.from_service_account_info")
@mock.patch("airflow.providers.google.cloud.utils.credentials_provider._SecretManagerClient")
diff --git a/tests/providers/google/common/hooks/test_base_google.py
b/tests/providers/google/common/hooks/test_base_google.py
index d2bf94767d..6ca5f19fc1 100644
--- a/tests/providers/google/common/hooks/test_base_google.py
+++ b/tests/providers/google/common/hooks/test_base_google.py
@@ -363,6 +363,7 @@ class TestGoogleBaseHook:
mock_get_creds_and_proj_id.assert_called_once_with(
key_path=None,
keyfile_dict=None,
+ credential_config_file=None,
key_secret_name=None,
key_secret_project_id=None,
scopes=self.instance.scopes,
@@ -399,6 +400,7 @@ class TestGoogleBaseHook:
mock_get_creds_and_proj_id.assert_called_once_with(
key_path="KEY_PATH.json",
keyfile_dict=None,
+ credential_config_file=None,
key_secret_name=None,
key_secret_project_id=None,
scopes=self.instance.scopes,
@@ -428,6 +430,7 @@ class TestGoogleBaseHook:
mock_get_creds_and_proj_id.assert_called_once_with(
key_path=None,
keyfile_dict=service_account,
+ credential_config_file=None,
key_secret_name=None,
key_secret_project_id=None,
scopes=self.instance.scopes,
@@ -447,6 +450,7 @@ class TestGoogleBaseHook:
mock_get_creds_and_proj_id.assert_called_once_with(
key_path=None,
keyfile_dict=None,
+ credential_config_file=None,
key_secret_name=None,
key_secret_project_id=None,
scopes=self.instance.scopes,
@@ -482,6 +486,7 @@ class TestGoogleBaseHook:
mock_get_creds_and_proj_id.assert_called_once_with(
key_path=None,
keyfile_dict=None,
+ credential_config_file=None,
key_secret_name=None,
key_secret_project_id=None,
scopes=self.instance.scopes,
@@ -683,6 +688,7 @@ class TestGoogleBaseHook:
mock_get_creds_and_proj_id.assert_called_once_with(
key_path=None,
keyfile_dict=None,
+ credential_config_file=None,
key_secret_name=None,
key_secret_project_id=None,
scopes=self.instance.scopes,