pingzh commented on a change in pull request #21445:
URL: https://github.com/apache/airflow/pull/21445#discussion_r806246555
##########
File path: airflow/serialization/serialized_objects.py
##########
@@ -707,6 +710,9 @@ def deserialize_operator(cls, encoded_op: Dict[str, Any])
-> Union[BaseOperator,
v = cls._deserialize_timedelta(v)
elif k in encoded_op["template_fields"]:
pass
+ elif k == "resources" and
Resources.is_operator_resources_json_string(v):
Review comment:
`# there are other values with the same key as `resources` so adding
this check`, with more thoughts, i think we can remove it.
##########
File path: tests/serialization/test_dag_serialization.py
##########
@@ -1174,6 +1175,24 @@ def test_operator_deserialize_old_names(self):
serialized_op = SerializedBaseOperator.deserialize_operator(blob)
assert serialized_op.downstream_task_ids == {'foo'}
+ def test_task_resources(self):
+ """
+ Test TaskGroup serialization/deserialization.
+ """
+ from airflow.operators.dummy import DummyOperator
+
+ execution_date = datetime(2020, 1, 1)
+ task_id = 'task1'
+ with DAG("test_task_resources", start_date=execution_date) as dag:
+ task = DummyOperator(task_id=task_id, resources={"cpus": 0.1,
"ram": 2048})
+
+ SerializedDAG.validate_schema(SerializedDAG.to_dict(dag))
+
Review comment:
> checks the json/dict structure
@ashb do you mean check the json/dict structure of the task.resource? or
the whole serialized dag json/dict?
##########
File path: tests/serialization/test_dag_serialization.py
##########
@@ -1174,6 +1175,24 @@ def test_operator_deserialize_old_names(self):
serialized_op = SerializedBaseOperator.deserialize_operator(blob)
assert serialized_op.downstream_task_ids == {'foo'}
+ def test_task_resources(self):
+ """
+ Test TaskGroup serialization/deserialization.
+ """
+ from airflow.operators.dummy import DummyOperator
+
+ execution_date = datetime(2020, 1, 1)
+ task_id = 'task1'
+ with DAG("test_task_resources", start_date=execution_date) as dag:
+ task = DummyOperator(task_id=task_id, resources={"cpus": 0.1,
"ram": 2048})
+
+ SerializedDAG.validate_schema(SerializedDAG.to_dict(dag))
+
Review comment:
thanks.
##########
File path: tests/serialization/test_dag_serialization.py
##########
@@ -1654,6 +1673,28 @@ def test_mapped_operator_xcomarg_serde():
assert xcom_arg.operator is serialized_dag.task_dict['op1']
+def test_task_resources_serde():
+ """
+ Test task resources serialization/deserialization.
+ """
+ from airflow.operators.dummy import DummyOperator
+
+ execution_date = datetime(2020, 1, 1)
+ task_id = 'task1'
+ with DAG("test_task_resources", start_date=execution_date) as _:
+ task = DummyOperator(task_id=task_id, resources={"cpus": 0.1, "ram":
2048})
+
+ serialized = SerializedBaseOperator._serialize(task)
+ assert serialized['resources'] == (
+ '{'
+ '"cpus": {"name": "CPU", "qty": 0.1, "units_str": "core(s)"}, '
+ '"disk": {"name": "Disk", "qty": 512, "units_str": "MB"}, '
+ '"gpus": {"name": "GPU", "qty": 0, "units_str": "gpu(s)"}, '
+ '"ram": {"name": "RAM", "qty": 2048, "units_str": "MB"}'
+ '}'
+ )
Review comment:
this depends on the protocol,
https://github.com/apache/airflow/pull/21445/files#diff-807ca0a4fd53aeb41166621c9842b0f89b7931fc64e9a60befa36c776db45efaR322
we can also use `to_dict` when serializing `resources`.
##########
File path: tests/serialization/test_dag_serialization.py
##########
@@ -1654,6 +1673,28 @@ def test_mapped_operator_xcomarg_serde():
assert xcom_arg.operator is serialized_dag.task_dict['op1']
+def test_task_resources_serde():
+ """
+ Test task resources serialization/deserialization.
+ """
+ from airflow.operators.dummy import DummyOperator
+
+ execution_date = datetime(2020, 1, 1)
+ task_id = 'task1'
+ with DAG("test_task_resources", start_date=execution_date) as _:
+ task = DummyOperator(task_id=task_id, resources={"cpus": 0.1, "ram":
2048})
+
+ serialized = SerializedBaseOperator._serialize(task)
+ assert serialized['resources'] == (
+ '{'
+ '"cpus": {"name": "CPU", "qty": 0.1, "units_str": "core(s)"}, '
+ '"disk": {"name": "Disk", "qty": 512, "units_str": "MB"}, '
+ '"gpus": {"name": "GPU", "qty": 0, "units_str": "gpu(s)"}, '
+ '"ram": {"name": "RAM", "qty": 2048, "units_str": "MB"}'
+ '}'
+ )
Review comment:
with more thoughts, it is better to use to_dict. changed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
