This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 2a975ea5b9 AIP-44 Fix serialization issues in Internal API. (#34003)
2a975ea5b9 is described below
commit 2a975ea5b93ca8fada0f5feadc809c5a47812473
Author: mhenc <[email protected]>
AuthorDate: Fri Sep 1 15:47:23 2023 +0200
AIP-44 Fix serialization issues in Internal API. (#34003)
* AIP-44 Fix serialization
* log exceptions
---
airflow/api_internal/endpoints/rpc_api_endpoint.py | 19 ++++++++++++-------
airflow/serialization/serialized_objects.py | 2 +-
tests/api_internal/endpoints/test_rpc_api_endpoint.py | 2 +-
tests/serialization/test_serialized_objects.py | 19 +++++++++++++++++--
4 files changed, 31 insertions(+), 11 deletions(-)
diff --git a/airflow/api_internal/endpoints/rpc_api_endpoint.py
b/airflow/api_internal/endpoints/rpc_api_endpoint.py
index 926c955190..6290b1749a 100644
--- a/airflow/api_internal/endpoints/rpc_api_endpoint.py
+++ b/airflow/api_internal/endpoints/rpc_api_endpoint.py
@@ -25,6 +25,7 @@ from typing import TYPE_CHECKING, Any, Callable
from flask import Response
from airflow.serialization.serialized_objects import BaseSerialization
+from airflow.utils.session import create_session
if TYPE_CHECKING:
from airflow.api_connexion.types import APIResponse
@@ -89,16 +90,20 @@ def internal_airflow_api(body: dict[str, Any]) ->
APIResponse:
if body.get("params"):
params_json = json.loads(str(body.get("params")))
params = BaseSerialization.deserialize(params_json,
use_pydantic_models=True)
- except Exception as err:
- log.error("Error (%s) when deserializing parameters: %s", err,
params_json)
+ except Exception as e:
+ log.error("Error when deserializing parameters for method: %s.",
method_name)
+ log.exception(e)
return Response(response="Error deserializing parameters.", status=400)
log.debug("Calling method %s.", method_name)
try:
- output = handler(**params)
- output_json = BaseSerialization.serialize(output,
use_pydantic_models=True)
- response = json.dumps(output_json) if output_json is not None else None
- return Response(response=response, headers={"Content-Type":
"application/json"})
+ # Session must be created there as it may be needed by serializer for
lazy-loaded fields.
+ with create_session() as session:
+ output = handler(**params, session=session)
+ output_json = BaseSerialization.serialize(output,
use_pydantic_models=True)
+ response = json.dumps(output_json) if output_json is not None else
None
+ return Response(response=response, headers={"Content-Type":
"application/json"})
except Exception as e:
- log.error("Error (%s) when calling method %s.", e, method_name)
+ log.error("Error executing method: %s.", method_name)
+ log.exception(e)
return Response(response=f"Error executing method: {method_name}.",
status=500)
diff --git a/airflow/serialization/serialized_objects.py
b/airflow/serialization/serialized_objects.py
index 230f4aca18..1ada95ea2c 100644
--- a/airflow/serialization/serialized_objects.py
+++ b/airflow/serialization/serialized_objects.py
@@ -493,7 +493,7 @@ class BaseSerialization:
def _pydantic_model_dump(model_cls: type[BaseModel], var: Any) ->
dict[str, Any]:
try:
- return model_cls.model_validate(var).model_dump() # type:
ignore[attr-defined]
+ return
model_cls.model_validate(var).model_dump(mode="json") # type:
ignore[attr-defined]
except AttributeError: # Pydantic 1.x compatibility.
return model_cls.from_orm(var).dict() # type:
ignore[attr-defined]
diff --git a/tests/api_internal/endpoints/test_rpc_api_endpoint.py
b/tests/api_internal/endpoints/test_rpc_api_endpoint.py
index a81d03f6ce..6a33b06a5c 100644
--- a/tests/api_internal/endpoints/test_rpc_api_endpoint.py
+++ b/tests/api_internal/endpoints/test_rpc_api_endpoint.py
@@ -123,7 +123,7 @@ class TestRpcApiEndpoint:
assert result_cmp_func(response_data, method_result)
- mock_test_method.assert_called_once_with(**method_params)
+ mock_test_method.assert_called_once_with(**method_params,
session=mock.ANY)
def test_method_with_exception(self):
mock_test_method.side_effect = ValueError("Error!!!")
diff --git a/tests/serialization/test_serialized_objects.py
b/tests/serialization/test_serialized_objects.py
index 17f5187579..9ac033672e 100644
--- a/tests/serialization/test_serialized_objects.py
+++ b/tests/serialization/test_serialized_objects.py
@@ -17,6 +17,9 @@
from __future__ import annotations
+import json
+from datetime import datetime
+
import pytest
from airflow.exceptions import SerializationError
@@ -89,14 +92,26 @@ def test_use_pydantic_models():
from airflow.serialization.serialized_objects import BaseSerialization
- ti = TaskInstance(task=EmptyOperator(task_id="task"), run_id="run_id",
state=State.RUNNING)
+ ti = TaskInstance(
+ task=EmptyOperator(task_id="task"),
+ run_id="run_id",
+ state=State.RUNNING,
+ )
+ start_date = datetime.utcnow()
+ ti.start_date = start_date
obj = [[ti]] # nested to verify recursive behavior
serialized = BaseSerialization.serialize(obj, use_pydantic_models=True) #
does not raise
deserialized = BaseSerialization.deserialize(serialized,
use_pydantic_models=True) # does not raise
-
assert isinstance(deserialized[0][0], TaskInstancePydantic)
+ serialized_json = json.dumps(serialized) # does not raise
+ deserialized_from_json = BaseSerialization.deserialize(
+ json.loads(serialized_json), use_pydantic_models=True
+ ) # does not raise
+ assert isinstance(deserialized_from_json[0][0], TaskInstancePydantic)
+ assert deserialized_from_json[0][0].start_date == start_date
+
def test_serialized_mapped_operator_unmap(dag_maker):
from airflow.serialization.serialized_objects import SerializedDAG