This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v2-11-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v2-11-test by this push:
new b68510efb25 [v2-11-test] Ensure that the the generated airflow.cfg
contains a ran… (#47755)
b68510efb25 is described below
commit b68510efb25e0e6ff6ae690173739e7462d99cb4
Author: Jens Scheffler <[email protected]>
AuthorDate: Wed Apr 30 21:20:46 2025 +0200
[v2-11-test] Ensure that the the generated airflow.cfg contains a ran…
(#47755)
Ensure that the the generated airflow.cfg contains a random jwt_secret and
fernet_key (#46966)
* Ensure that the the generated airflow.cfg contains a random jwt_secret
and fernet_key
I don't know exactly when this got broken, and unforutnaltey it wasn't
tested,
but I suspect it might have been around the time we swapped the default
config
from a config file to the yaml based values. I.e. a while ago!
To make sure it doesn't get broken I've gone and added some unit tests
And to make my next PR and test easier I have done the same thing with the
`auth_jwt_secret` that we do for fernet_key -- of only set it in the config
file if we're generating that file, not always in memory.
* Improve upgrade path by generating and warning about the missing config
(cherry picked from commit 106b8726724fa712176dfc17ae192dbf5be83e17)
Co-authored-by: Ash Berlin-Taylor <[email protected]>
* Missing another one...
* Fixup!
* Fix error about missing section
* Remove not-needed test
---------
(cherry picked from commit 106b8726724fa712176dfc17ae192dbf5be83e17)
Co-authored-by: Ash Berlin-Taylor <[email protected]>
---
airflow/api_internal/endpoints/rpc_api_endpoint.py | 4 ++--
airflow/api_internal/internal_api_call.py | 4 ++--
airflow/configuration.py | 10 ++++++---
airflow/utils/jwt_signer.py | 18 +++++++++++++++
airflow/utils/log/file_task_handler.py | 4 ++--
airflow/utils/serve_logs.py | 4 ++--
airflow/www/app.py | 3 ++-
tests/core/test_configuration.py | 26 ++++++++++++++++++++++
8 files changed, 61 insertions(+), 12 deletions(-)
diff --git a/airflow/api_internal/endpoints/rpc_api_endpoint.py
b/airflow/api_internal/endpoints/rpc_api_endpoint.py
index c3d8b671fbb..a3c3c48d82e 100644
--- a/airflow/api_internal/endpoints/rpc_api_endpoint.py
+++ b/airflow/api_internal/endpoints/rpc_api_endpoint.py
@@ -42,7 +42,7 @@ from airflow.models.taskinstance import
_record_task_map_for_downstreams
from airflow.models.xcom_arg import _get_task_map_length
from airflow.sensors.base import _orig_start_date
from airflow.serialization.serialized_objects import BaseSerialization
-from airflow.utils.jwt_signer import JWTSigner
+from airflow.utils.jwt_signer import JWTSigner, get_signing_key
from airflow.utils.session import create_session
if TYPE_CHECKING:
@@ -178,7 +178,7 @@ def internal_airflow_api(body: dict[str, Any]) ->
APIResponse:
auth = request.headers.get("Authorization", "")
clock_grace = conf.getint("core", "internal_api_clock_grace", fallback=30)
signer = JWTSigner(
- secret_key=conf.get("core", "internal_api_secret_key"),
+ secret_key=get_signing_key("core", "internal_api_secret_key"),
expiration_time_in_seconds=clock_grace,
leeway_in_seconds=clock_grace,
audience="api",
diff --git a/airflow/api_internal/internal_api_call.py
b/airflow/api_internal/internal_api_call.py
index 064834d7c86..f52b0323cbc 100644
--- a/airflow/api_internal/internal_api_call.py
+++ b/airflow/api_internal/internal_api_call.py
@@ -33,7 +33,7 @@ from airflow.configuration import conf
from airflow.exceptions import AirflowConfigException, AirflowException
from airflow.settings import _ENABLE_AIP_44,
force_traceback_session_for_untrusted_components
from airflow.typing_compat import ParamSpec
-from airflow.utils.jwt_signer import JWTSigner
+from airflow.utils.jwt_signer import JWTSigner, get_signing_key
PS = ParamSpec("PS")
RT = TypeVar("RT")
@@ -139,7 +139,7 @@ def internal_api_call(func: Callable[PS, RT]) ->
Callable[PS, RT]:
)
def make_jsonrpc_request(method_name: str, params_json: str) -> bytes:
signer = JWTSigner(
- secret_key=conf.get("core", "internal_api_secret_key"),
+ secret_key=get_signing_key("core", "internal_api_secret_key"),
expiration_time_in_seconds=conf.getint("core",
"internal_api_clock_grace", fallback=30),
audience="api",
)
diff --git a/airflow/configuration.py b/airflow/configuration.py
index afb4b5f3808..3affeb3c24d 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -1949,7 +1949,7 @@ def get_airflow_config(airflow_home: str) -> str:
def get_all_expansion_variables() -> dict[str, Any]:
- return {k: v for d in [globals(), locals()] for k, v in d.items()}
+ return {k: v for d in [globals(), locals()] for k, v in d.items() if not
k.startswith("_")}
def _generate_fernet_key() -> str:
@@ -2000,6 +2000,7 @@ def create_pre_2_7_defaults() -> ConfigParser:
def write_default_airflow_configuration_if_needed() -> AirflowConfigParser:
+ global FERNET_KEY
airflow_config = pathlib.Path(AIRFLOW_CONFIG)
if airflow_config.is_dir():
msg = (
@@ -2023,13 +2024,16 @@ def write_default_airflow_configuration_if_needed() ->
AirflowConfigParser:
raise FileNotFoundError(msg) from None
log.debug("Create directory %r for Airflow config",
config_directory.__fspath__())
config_directory.mkdir(parents=True, exist_ok=True)
- if conf.get("core", "fernet_key", fallback=None) is None:
+ if conf.get("core", "fernet_key", fallback=None) in (None, ""):
# We know that FERNET_KEY is not set, so we can generate it, set
as global key
# and also write it to the config file so that same key will be
used next time
- global FERNET_KEY
FERNET_KEY = _generate_fernet_key()
conf.remove_option("core", "fernet_key")
+ if not conf.has_section("core"):
+ conf.add_section("core")
conf.set("core", "fernet_key", FERNET_KEY)
+
conf.configuration_description["core"]["options"]["fernet_key"]["default"] =
FERNET_KEY
+
pathlib.Path(airflow_config.__fspath__()).touch()
make_group_other_inaccessible(airflow_config.__fspath__())
with open(airflow_config, "w") as file:
diff --git a/airflow/utils/jwt_signer.py b/airflow/utils/jwt_signer.py
index fe4811eb827..5d8e82965b3 100644
--- a/airflow/utils/jwt_signer.py
+++ b/airflow/utils/jwt_signer.py
@@ -16,6 +16,9 @@
# under the License.
from __future__ import annotations
+import logging
+import os
+from base64 import b64encode
from datetime import timedelta
from typing import Any
@@ -24,6 +27,21 @@ import jwt
from airflow.utils import timezone
+def get_signing_key(section: str, key: str) -> str:
+ from airflow.configuration import conf
+
+ secret_key = conf.get(section, key, fallback="")
+
+ if secret_key == "":
+ logging.getLogger(__name__).warning(
+ "`%s/%s` was empty, using a generated one for now. Please set this
in your config", section, key
+ )
+ secret_key = b64encode(os.urandom(16)).decode("utf-8")
+ # Set it back so any other callers get the same value for the duration
of this process
+ conf.set(section, key, secret_key)
+ return secret_key
+
+
class JWTSigner:
"""
Signs and verifies JWT Token. Used to authorise and verify requests.
diff --git a/airflow/utils/log/file_task_handler.py
b/airflow/utils/log/file_task_handler.py
index 2df73ef4ffa..a33ab73afc7 100644
--- a/airflow/utils/log/file_task_handler.py
+++ b/airflow/utils/log/file_task_handler.py
@@ -90,11 +90,11 @@ def _fetch_logs_from_service(url, log_relative_path):
# Import occurs in function scope for perf. Ref:
https://github.com/apache/airflow/pull/21438
import requests
- from airflow.utils.jwt_signer import JWTSigner
+ from airflow.utils.jwt_signer import JWTSigner, get_signing_key
timeout = conf.getint("webserver", "log_fetch_timeout_sec", fallback=None)
signer = JWTSigner(
- secret_key=conf.get("webserver", "secret_key"),
+ secret_key=get_signing_key("webserver", "secret_key"),
expiration_time_in_seconds=conf.getint("webserver",
"log_request_clock_grace", fallback=30),
audience="task-instance-logs",
)
diff --git a/airflow/utils/serve_logs.py b/airflow/utils/serve_logs.py
index 31ef86600da..2cdb0bc5851 100644
--- a/airflow/utils/serve_logs.py
+++ b/airflow/utils/serve_logs.py
@@ -37,7 +37,7 @@ from werkzeug.exceptions import HTTPException
from airflow.configuration import conf
from airflow.utils.docs import get_docs_url
-from airflow.utils.jwt_signer import JWTSigner
+from airflow.utils.jwt_signer import JWTSigner, get_signing_key
from airflow.utils.module_loading import import_string
logger = logging.getLogger(__name__)
@@ -71,7 +71,7 @@ def create_app():
except Exception as e:
raise ImportError(f"Unable to load {log_config_class} due to
error: {e}")
signer = JWTSigner(
- secret_key=conf.get("webserver", "secret_key"),
+ secret_key=get_signing_key("webserver", "secret_key"),
expiration_time_in_seconds=expiration_time_in_seconds,
audience="task-instance-logs",
)
diff --git a/airflow/www/app.py b/airflow/www/app.py
index 23d79b01861..927d3c9b693 100644
--- a/airflow/www/app.py
+++ b/airflow/www/app.py
@@ -35,6 +35,7 @@ from airflow.logging_config import configure_logging
from airflow.models import import_all_models
from airflow.settings import _ENABLE_AIP_44
from airflow.utils.json import AirflowJsonProvider
+from airflow.utils.jwt_signer import get_signing_key
from airflow.www.extensions.init_appbuilder import init_appbuilder
from airflow.www.extensions.init_appbuilder_links import init_appbuilder_links
from airflow.www.extensions.init_auth_manager import get_auth_manager
@@ -73,7 +74,7 @@ csrf = CSRFProtect()
def create_app(config=None, testing=False):
"""Create a new instance of Airflow WWW app."""
flask_app = Flask(__name__)
- flask_app.secret_key = conf.get("webserver", "SECRET_KEY")
+ flask_app.secret_key = get_signing_key("webserver", "SECRET_KEY")
flask_app.config["PERMANENT_SESSION_LIFETIME"] =
timedelta(minutes=settings.get_session_lifetime_config())
diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py
index 58e1e029af8..d48c4665310 100644
--- a/tests/core/test_configuration.py
+++ b/tests/core/test_configuration.py
@@ -1800,3 +1800,29 @@ class TestWriteDefaultAirflowConfigurationIfNeeded:
mock_mask_secret.assert_any_call("supersecret2")
assert mock_mask_secret.call_count == 2
+
+
+@conf_vars({("core", "unit_test_mode"): "False"})
+def test_write_default_config_contains_generated_secrets(tmp_path,
monkeypatch):
+ import airflow.configuration
+
+ cfgpath = tmp_path / "airflow-gneerated.cfg"
+ # Patch these globals so it gets reverted by monkeypath after this test is
over.
+ monkeypatch.setattr(airflow.configuration, "FERNET_KEY", "")
+ monkeypatch.setattr(airflow.configuration, "AIRFLOW_CONFIG", str(cfgpath))
+
+ # Create a new global conf object so our changes don't persist
+ localconf: AirflowConfigParser = airflow.configuration.initialize_config()
+ monkeypatch.setattr(airflow.configuration, "conf", localconf)
+
+ airflow.configuration.write_default_airflow_configuration_if_needed()
+
+ assert cfgpath.is_file()
+
+ lines = cfgpath.read_text().splitlines()
+
+ assert airflow.configuration.FERNET_KEY
+
+ fernet_line = next(line for line in lines if line.startswith("fernet_key =
"))
+
+ assert fernet_line == f"fernet_key = {airflow.configuration.FERNET_KEY}"
