kaxil commented on code in PR #74421:
URL: https://github.com/apache/airflow/pull/74421#discussion_r4551149580
##########
task-sdk/src/airflow/sdk/execution_time/context.py:
##########
@@ -163,8 +163,23 @@ def _wrap_external_ref(ref: str) -> dict[str, JsonValue]:
return {_EXTERNAL_STATE_REF_KEY: ref}
-def _unwrap_external_ref(stored: dict) -> str | None:
- return stored.get(_EXTERNAL_STATE_REF_KEY)
+def _unwrap_external_ref(stored: JsonValue) -> str | None:
+ """Return the reference inside a ``{marker: ref}`` wrapper, or ``None``
for any other value."""
+ if isinstance(stored, dict) and stored.keys() == {_EXTERNAL_STATE_REF_KEY}:
+ ref = stored[_EXTERNAL_STATE_REF_KEY]
+ if isinstance(ref, str):
+ return ref
+ return None
+
+
+def _validate_state_store_value(value: JsonValue) -> None:
+ if value is None:
+ raise ValueError("Cannot set value as None")
+ # get() would read such a value back as a backend reference
Review Comment:
Applied, but kept the exact-wrapper case in the same comment, since that's
the shape new SDKs still read as a ref.
##########
providers/common/io/tests/unit/common/io/state_store/test_backend.py:
##########
@@ -286,20 +286,20 @@ def test_asset_serialize_offloads_to_storage(self,
asset_scope, base_path):
ref = store.serialize_asset_state_store_to_ref(value={"x": 1},
key="k", scope=asset_scope)
assert ref.startswith("file://")
- def test_task_serialize_to_db_when_below_threshold(self, task_scope,
base_path):
- with conf_vars(
- {
- ("common.io", "state_store_objectstorage_path"): base_path,
- ("common.io", "state_store_objectstorage_threshold"): "10000",
- }
- ):
- backend._get_threshold.cache_clear()
- store = StateStoreObjectStorageBackend()
- ref = store.serialize_task_state_store_to_ref(value={"x": 1},
key="k", scope=task_scope)
- assert not ref.startswith("file://")
- assert store.deserialize_task_state_store_from_ref(ref) == {"x": 1}
-
- def test_asset_serialize_to_db_when_below_threshold(self, asset_scope,
base_path):
+ @pytest.mark.parametrize(
+ ("kind", "scope_fixture"),
+ [pytest.param("task", "task_scope", id="task"), pytest.param("asset",
"asset_scope", id="asset")],
+ )
+ @pytest.mark.parametrize(
+ "base_default",
+ [pytest.param(None, id="sdk-keeps-inline"), pytest.param('{"x": 1}',
id="sdk-wraps-every-return")],
+ )
+ def test_serialize_below_threshold_returns_base_default(
Review Comment:
Renamed.
##########
task-sdk/tests/task_sdk/execution_time/test_context.py:
##########
@@ -1962,6 +1963,87 @@ def
test_get_with_custom_backend_removes_decoration_marker(self, mock_supervisor
assert result == {"rows": 123}
backend.deserialize_task_state_store_from_ref.assert_called_once_with("s3://bucket/ti_123/job_id")
+ def test_set_with_custom_backend_returning_none_stores_value_inline(self,
mock_supervisor_comms):
+ """A backend that returns None keeps the value inline, with no marker
around it."""
+ mock_supervisor_comms.send.return_value = OKResponse(ok=True)
+
+ backend = MagicMock(spec=BaseStoreBackend)
+ backend.serialize_task_state_store_to_ref.return_value = None
+
+ with (
+
patch("airflow.sdk.execution_time.context._get_worker_state_store_backend",
return_value=backend),
Review Comment:
Done, on all the new backend tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
