This is an automated email from the ASF dual-hosted git repository.
kaxil pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 537feafb369 Fix Variables API handling of non-string JSON values
(#71018)
537feafb369 is described below
commit 537feafb3697338b3988d826b9c9b45da06b1f23
Author: Kaxil Naik <[email protected]>
AuthorDate: Tue Aug 11 22:54:30 2026 +0100
Fix Variables API handling of non-string JSON values (#71018)
POST /api/v2/variables with a non-string JSON value (array, object,
bool, null) silently stored the Python repr of the value, which cannot
be read back with deserialize_json. PATCH with the same payload failed
with a masked 500 because the raw value hit the ORM Fernet encryption
step.
VariableBody now JSON-encodes non-string values once at request
validation, covering POST, PATCH and bulk consistently, matching the
behaviour the bulk create path already had for dicts and lists.
Variable.set_val also raises a clear TypeError instead of the cryptic
"encoding without a string argument".
---
.../api_fastapi/core_api/datamodels/variables.py | 15 +++-
.../core_api/services/public/variables.py | 3 +-
airflow-core/src/airflow/models/variable.py | 9 ++-
.../core_api/routes/public/test_variables.py | 91 ++++++++++++++++++++--
airflow-core/tests/unit/models/test_variable.py | 5 ++
5 files changed, 112 insertions(+), 11 deletions(-)
diff --git
a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/variables.py
b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/variables.py
index 75fefdd656b..16c98c8588a 100644
--- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/variables.py
+++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/variables.py
@@ -20,7 +20,7 @@ from __future__ import annotations
import json
from collections.abc import Iterable
-from pydantic import Field, JsonValue, model_validator
+from pydantic import Field, JsonValue, field_validator, model_validator
from airflow._shared.secrets_masker import redact
from airflow.api_fastapi.core_api.base import BaseModel, StrictBaseModel,
make_partial_model
@@ -61,6 +61,19 @@ class VariableBody(StrictBaseModel):
description: str | None = Field(default=None)
team_name: str | None = Field(max_length=50, default=None)
+ @field_validator("value", mode="after")
+ @classmethod
+ def serialize_non_string_value(cls, value: JsonValue) -> str:
+ # Variables are stored as strings. A non-string JSON value (list,
dict, number, bool,
+ # null) must be JSON-encoded here: POST would otherwise store its
Python repr, which
+ # deserialize_json cannot read back, and PATCH would fail in the ORM
Fernet step on
+ # bytes(value, "utf-8") (see #71010). indent=2 matches
Variable.set(serialize_json=True).
+ # Not allow_nan=False: rejecting here yields a 422 whose response
embeds the NaN input,
+ # which starlette's JSONResponse cannot serialize -- the request would
500 instead.
+ if isinstance(value, str):
+ return value
+ return json.dumps(value, indent=2)
+
@model_validator(mode="after")
def validate_team_name(self) -> VariableBody:
if self.team_name is not None and not conf.getboolean("core",
"multi_team"):
diff --git
a/airflow-core/src/airflow/api_fastapi/core_api/services/public/variables.py
b/airflow-core/src/airflow/api_fastapi/core_api/services/public/variables.py
index 95d52bfd805..37941fed550 100644
--- a/airflow-core/src/airflow/api_fastapi/core_api/services/public/variables.py
+++ b/airflow-core/src/airflow/api_fastapi/core_api/services/public/variables.py
@@ -126,13 +126,12 @@ class BulkVariableService(BulkService[VariableBody]):
for variable in action.entities:
if variable.key in create_keys:
- should_serialize_json = isinstance(variable.value, (dict,
list))
+ # VariableBody already JSON-encodes non-string values, so
no serialize_json here.
Variable.set(
key=variable.key,
value=variable.value,
description=variable.description,
session=self.session,
- serialize_json=should_serialize_json,
)
results.success.append(variable.key)
diff --git a/airflow-core/src/airflow/models/variable.py
b/airflow-core/src/airflow/models/variable.py
index b06e73cd5f5..c366a2482b1 100644
--- a/airflow-core/src/airflow/models/variable.py
+++ b/airflow-core/src/airflow/models/variable.py
@@ -111,6 +111,11 @@ class Variable(Base, LoggingMixin):
def set_val(self, value):
"""Encode the specified value with Fernet Key and store it in
Variables Table."""
if value is not None:
+ if not isinstance(value, str):
+ raise TypeError(
+ f"Variable value must be a string, got
{type(value).__name__}. "
+ "Use Variable.set(key, value, serialize_json=True) to
store non-string values."
+ )
fernet = get_fernet()
self._val = fernet.encrypt(bytes(value, "utf-8")).decode()
self.is_encrypted = fernet.is_encrypted
@@ -214,7 +219,9 @@ class Variable(Base, LoggingMixin):
This operation overwrites an existing variable using the session's
dialect-specific upsert operation.
:param key: Variable Key
- :param value: Value to set for the Variable
+ :param value: Value to set for the Variable. Non-string values are
coerced with ``str()``
+ unless ``serialize_json=True``, so pass ``serialize_json=True`` to
store them as JSON
+ that ``deserialize_json=True`` can read back.
:param description: Description of the Variable
:param serialize_json: Serialize the value to a JSON string
:param team_name: Team name associated to the variable (if any)
diff --git
a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_variables.py
b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_variables.py
index 234bbe6b1f7..1198f748481 100644
---
a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_variables.py
+++
b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_variables.py
@@ -546,6 +546,29 @@ class TestPatchVariable(TestVariableEndpoint):
}
check_last_log(session, dag_id=None, event="patch_variable",
logical_date=None)
+ @pytest.mark.parametrize(
+ "value",
+ [
+ ["a", "b"],
+ {"name": "test", "id": 123, "active": True},
+ 42,
+ 4.2,
+ True,
+ None,
+ ],
+ ids=["list", "dict", "int", "float", "bool", "none"],
+ )
+ def test_patch_non_string_json_value_round_trips(self, test_client, value):
+ """Non-string JSON values must be stored as valid JSON, not raise a
500 (issue #71010)."""
+ self.create_variables()
+ body = {"key": TEST_VARIABLE_KEY, "value": value, "description":
TEST_VARIABLE_DESCRIPTION}
+
+ response = test_client.patch(f"/variables/{TEST_VARIABLE_KEY}",
json=body)
+
+ assert response.status_code == 200
+ assert json.loads(response.json()["value"]) == value
+ assert Variable.get(TEST_VARIABLE_KEY, deserialize_json=True) == value
+
def test_patch_should_respond_400(self, test_client):
response = test_client.patch(
f"/variables/{TEST_VARIABLE_KEY}",
@@ -681,6 +704,30 @@ class TestPostVariable(TestVariableEndpoint):
assert response.json() == expected_response
check_last_log(session, dag_id=None, event="post_variable",
logical_date=None)
+ @pytest.mark.parametrize(
+ "value",
+ [
+ ["a", "b"],
+ {"name": "test", "id": 123, "active": True},
+ 42,
+ 4.2,
+ True,
+ None,
+ ],
+ ids=["list", "dict", "int", "float", "bool", "none"],
+ )
+ def test_post_non_string_json_value_round_trips(self, test_client, value):
+ """Non-string JSON values must be stored as valid JSON, not a Python
repr (issue #71010)."""
+ body = {"key": "non_string_value", "value": value, "description":
"non-string JSON value"}
+
+ response = test_client.post("/variables", json=body)
+
+ assert response.status_code == 201
+ assert json.loads(response.json()["value"]) == value
+ stored_raw = Variable.get("non_string_value")
+ assert json.loads(stored_raw) == value
+ assert Variable.get("non_string_value", deserialize_json=True) == value
+
def test_post_with_team_should_respond_201(self, test_client,
testing_team, session):
self.create_variables()
body = {
@@ -1395,11 +1442,15 @@ class TestBulkVariables(TestVariableEndpoint):
),
("my_list_var_param", ["alpha", 42, False, {"nested": "item
param"}], "A list value (param)"),
("my_string_var_param", "plain string param", "A plain string
(param)"),
+ ("my_bool_var_param", True, "A bool value (param)"),
+ ("my_none_var_param", None, "A null value (param)"),
],
ids=[
"dict_variable",
"list_variable",
"string_variable",
+ "bool_variable",
+ "none_variable",
],
)
def test_bulk_create_entity_serialization(
@@ -1420,20 +1471,46 @@ class TestBulkVariables(TestVariableEndpoint):
response = test_client.patch("/variables", json=actions)
assert response.status_code == 200
- if isinstance(entity_value, (dict, list)):
- retrieved_value_deserialized = Variable.get(entity_key,
deserialize_json=True)
- assert retrieved_value_deserialized == entity_value
- retrieved_value_raw_string = Variable.get(entity_key,
deserialize_json=False)
- assert retrieved_value_raw_string == json.dumps(entity_value,
indent=2)
- else:
+ if isinstance(entity_value, str):
+ # Strings are stored verbatim; a plain non-JSON string is not
deserializable.
retrieved_value_raw = Variable.get(entity_key,
deserialize_json=False)
- assert retrieved_value_raw == str(entity_value)
+ assert retrieved_value_raw == entity_value
with pytest.raises(json.JSONDecodeError):
Variable.get(entity_key, deserialize_json=True)
+ else:
+ retrieved_value_deserialized = Variable.get(entity_key,
deserialize_json=True)
+ assert retrieved_value_deserialized == entity_value
+ retrieved_value_raw_string = Variable.get(entity_key,
deserialize_json=False)
+ assert retrieved_value_raw_string == json.dumps(entity_value,
indent=2)
check_last_log(session, dag_id=None, event="bulk_variables",
logical_date=None)
+ def test_bulk_update_non_string_json_value_round_trips(self, test_client):
+ """A non-string value in bulk update must not 500 the whole request
(issue #71010)."""
+ self.create_variables()
+ actions = {
+ "actions": [
+ {
+ "action": "update",
+ "entities": [
+ {
+ "key": TEST_VARIABLE_KEY,
+ "value": {"nested": [1, 2, True, None]},
+ "description": TEST_VARIABLE_DESCRIPTION,
+ }
+ ],
+ "action_on_non_existence": "fail",
+ }
+ ]
+ }
+
+ response = test_client.patch("/variables", json=actions)
+
+ assert response.status_code == 200
+ assert response.json()["update"] == {"success": [TEST_VARIABLE_KEY],
"errors": []}
+ assert Variable.get(TEST_VARIABLE_KEY, deserialize_json=True) ==
{"nested": [1, 2, True, None]}
+
def test_bulk_variables_should_respond_401(self,
unauthenticated_test_client):
response = unauthenticated_test_client.patch("/variables", json={})
assert response.status_code == 401
diff --git a/airflow-core/tests/unit/models/test_variable.py
b/airflow-core/tests/unit/models/test_variable.py
index e56ee51d734..001a29a7c0c 100644
--- a/airflow-core/tests/unit/models/test_variable.py
+++ b/airflow-core/tests/unit/models/test_variable.py
@@ -127,6 +127,11 @@ class TestVariable:
Variable.set("tested_var_set_id", "Monday morning breakfast")
assert Variable.get("tested_var_set_id") == "Monday morning breakfast"
+ def test_set_val_rejects_non_string_with_clear_error(self):
+ var = Variable(key="a_key")
+ with pytest.raises(TypeError, match="Variable value must be a string,
got list"):
+ var.val = ["a", "b"]
+
def test_variable_set_with_env_variable(self, caplog, session):
caplog.set_level(logging.WARNING, logger=variable.log.name)
Variable.set(key="key", value="db-value", session=session)
