ashb commented on a change in pull request #9645:
URL: https://github.com/apache/airflow/pull/9645#discussion_r449642855
##########
File path: airflow/configuration.py
##########
@@ -115,7 +117,9 @@ class AirflowConfigParser(ConfigParser):
# These configuration elements can be fetched as the stdout of commands
# following the "{section}__{name}__cmd" pattern, the idea behind this
# is to not store password on boxes in text files.
- as_command_stdout = {
+ # These configs can also be fetched from Secrets backend
+ # following the "{section}__{name}__secret" pattern
+ configs_with_secret = {
Review comment:
```suggestion
senstive_config_values = {
```
##########
File path: airflow/providers/amazon/aws/secrets/secrets_manager.py
##########
@@ -62,13 +67,15 @@ def __init__(
self,
connections_prefix: str = 'airflow/connections',
variables_prefix: str = 'airflow/variables',
+ configurations_prefix: str = 'airflow/configuration',
Review comment:
```suggestion
config_prefix: str = 'airflow/config',
```
##########
File path: airflow/secrets/base_secrets.py
##########
@@ -73,3 +73,12 @@ def get_variable(self, key: str) -> Optional[str]:
:return: Variable Value
"""
raise NotImplementedError()
+
+ def get_configuration(self, key: str) -> Optional[str]:
Review comment:
```suggestion
def get_configuration(self, section: str, key: str) -> Optional[str]:
```
##########
File path: airflow/providers/amazon/aws/secrets/secrets_manager.py
##########
@@ -62,13 +67,15 @@ def __init__(
self,
connections_prefix: str = 'airflow/connections',
variables_prefix: str = 'airflow/variables',
+ configurations_prefix: str = 'airflow/configuration',
Review comment:
etc (haven't made this change everywhere)
##########
File path: airflow/providers/amazon/aws/secrets/secrets_manager.py
##########
@@ -42,8 +42,11 @@ class SecretsManagerBackend(BaseSecretsBackend,
LoggingMixin):
For example, if secrets prefix is ``airflow/connections/smtp_default``,
this would be accessible
if you provide ``{"connections_prefix": "airflow/connections"}`` and
request conn_id ``smtp_default``.
- And if variables prefix is ``airflow/variables/hello``, this would be
accessible
+ If variables prefix is ``airflow/variables/hello``, this would be
accessible
if you provide ``{"variables_prefix": "airflow/variables"}`` and request
variable key ``hello``.
+ And if configurations_prefix is
``airflow/configurations/sql_alchemy_conn``, this would be accessible
Review comment:
```suggestion
And if configurations_prefix is
``airflow/config/core/sql_alchemy_conn``, this would be accessible
```
##########
File path: airflow/configuration.py
##########
@@ -267,17 +271,49 @@ def _get_env_var_option(self, section, key):
env_var_cmd = env_var + '_CMD'
if env_var_cmd in os.environ:
# if this is a valid command key...
- if (section, key) in self.as_command_stdout:
+ if (section, key) in self.configs_with_secret:
return run_command(os.environ[env_var_cmd])
def _get_cmd_option(self, section, key):
fallback_key = key + '_cmd'
# if this is a valid command key...
- if (section, key) in self.as_command_stdout:
+ if (section, key) in self.configs_with_secret:
if super().has_option(section, fallback_key):
command = super().get(section, fallback_key)
return run_command(command)
+ @cached_property
+ def _secrets_backend_client(self):
+ if super().has_option('secrets', 'backend') is False:
+ return None
+
+ secrets_backend_cls = super().get('secrets', 'backend')
+ if not secrets_backend_cls:
+ return None
+ print("secrets_backend_cls", secrets_backend_cls)
+ secrets_backend = import_string(secrets_backend_cls)
+ if secrets_backend:
+ try:
+ alternative_secrets_config_dict = json.loads(
+ super().get('secrets', 'backend_kwargs', fallback='{}')
+ )
+ except json.JSONDecodeError:
+ alternative_secrets_config_dict = {}
+ return secrets_backend(**alternative_secrets_config_dict)
Review comment:
Hmmmm, not a huge fan of the duplication here and in
`airflow/secrets/__init__.py`.
Since this would only be called _after_ the class is constructed would we
instead do
```suggestion
@cached_property
def _secrets_backend_client(self):
if super().has_option('secrets', 'backend') is False:
return None
from airflow.secrets import secrets_backend_list
return secrets_backend_list
```
(give or take).
By delaying the import to inside the function no import loop is created.
This probably also means adding a `get_config(section, key)` to
airflow.secrets, and possibly changing the default implemenation from
`NotImplementedError` to `return None`.
Note: we should pass they key and section in directly, and let the secret
backends convert that to a single parameter _if needed_ (for instance we could
have /config/core/sql_alchemy_conn, rather than config/core__sql_alchemy_conn)
##########
File path: airflow/providers/amazon/aws/secrets/secrets_manager.py
##########
@@ -42,8 +42,11 @@ class SecretsManagerBackend(BaseSecretsBackend,
LoggingMixin):
For example, if secrets prefix is ``airflow/connections/smtp_default``,
this would be accessible
if you provide ``{"connections_prefix": "airflow/connections"}`` and
request conn_id ``smtp_default``.
- And if variables prefix is ``airflow/variables/hello``, this would be
accessible
+ If variables prefix is ``airflow/variables/hello``, this would be
accessible
if you provide ``{"variables_prefix": "airflow/variables"}`` and request
variable key ``hello``.
+ And if configurations_prefix is
``airflow/configurations/sql_alchemy_conn``, this would be accessible
+ if you provide ``{"configurations_prefix": "airflow/configurations"}`` and
request variable
Review comment:
```suggestion
if you provide ``{"configurations_prefix": "airflow/config"}`` and
request variable
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
