ashb commented on a change in pull request #6715: [AIRFLOW-5945] Make inbuilt
OperatorLinks work when using Serialization
URL: https://github.com/apache/airflow/pull/6715#discussion_r354555281
##########
File path: tests/serialization/test_dag_serialization.py
##########
@@ -381,6 +362,177 @@ def test_roundtrip_relativedelta(self, val, expected):
round_tripped = SerializedDAG._deserialize(serialized)
self.assertEqual(val, round_tripped)
+ def test_extra_serialized_field_and_operator_links(self):
+ """
+ Assert extra field exists & OperatorLinks defined in Plugins and
inbuilt Operator Links.
+
+ This tests also depends on GoogleLink() registered as a plugin
+ in tests/plugins/test_plugin.py
+
+ The function tests that if extra operator links are registered in
plugin
+ in ``operator_extra_links`` and the same is also defined in
+ the Operator in ``BaseOperator.operator_extra_links``, it has the
correct
+ extra link.
+ """
+ test_date = datetime(2019, 8, 1)
+ dag = DAG(dag_id='simple_dag', start_date=test_date)
+ CustomBaseOperator(task_id='simple_task', dag=dag, bash_command="true")
+
+ serialized_dag = SerializedDAG.to_dict(dag)
+ self.assertIn("bash_command", serialized_dag["dag"]["tasks"][0])
+
+ dag = SerializedDAG.from_dict(serialized_dag)
+ simple_task = dag.task_dict["simple_task"]
+ self.assertEqual(getattr(simple_task, "bash_command"), "true")
+
+ #########################################################
+ # Verify Operator Links work with Serialized Operator
+ #########################################################
+ # Check Serialized version of operator link only contains the inbuilt
Op Link
+ self.assertEqual(
+ serialized_dag["dag"]["tasks"][0]["_operator_extra_links"],
+ [{'airflow.utils.tests.CustomBaseOpLink': {}}]
+ )
+
+ # Test all the extra_links are set
+ self.assertCountEqual(simple_task.extra_links, ['Google Custom',
'airflow', 'github', 'google'])
+
+ ti = TaskInstance(task=simple_task, execution_date=test_date)
+ ti.xcom_push('search_query', "dummy_value_1")
+
+ # Test Deserialized inbuilt link
+ custom_inbuilt_link = simple_task.get_extra_links(test_date,
CustomBaseOpLink.name)
+
self.assertEqual('http://google.com/custom_base_link?search=dummy_value_1',
custom_inbuilt_link)
+
+ # Test Deserialized link registered via Airflow Plugin
+ google_link_from_plugin = simple_task.get_extra_links(test_date,
GoogleLink.name)
+ self.assertEqual("https://www.google.com", google_link_from_plugin)
+
+ def test_extra_serialized_field_and_multiple_operator_links(self):
+ """
+ Assert extra field exists & OperatorLinks defined in Plugins and
inbuilt Operator Links.
+
+ This tests also depends on GoogleLink() registered as a plugin
+ in tests/plugins/test_plugin.py
+
+ The function tests that if extra operator links are registered in
plugin
+ in ``operator_extra_links`` and the same is also defined in
+ the Operator in ``BaseOperator.operator_extra_links``, it has the
correct
+ extra link.
+ """
+ test_date = datetime(2019, 8, 1)
+ dag = DAG(dag_id='simple_dag', start_date=test_date)
+ CustomBaseOperator(task_id='simple_task', dag=dag,
+ bash_command=["echo", "true"])
+
+ serialized_dag = SerializedDAG.to_dict(dag)
+ self.assertIn("bash_command", serialized_dag["dag"]["tasks"][0])
+
+ dag = SerializedDAG.from_dict(serialized_dag)
+ simple_task = dag.task_dict["simple_task"]
+ self.assertEqual(getattr(simple_task, "bash_command"), ["echo",
"true"])
+
+ #########################################################
+ # Verify Operator Links work with Serialized Operator
+ #########################################################
+ # Check Serialized version of operator link only contains the inbuilt
Op Link
+ self.assertEqual(
+ serialized_dag["dag"]["tasks"][0]["_operator_extra_links"],
+ [
+ {'airflow.utils.tests.CustomBaseIndexOpLink': {'index': 0}},
+ {'airflow.utils.tests.CustomBaseIndexOpLink': {'index': 1}},
+ ]
+ )
+
+ # Test all the extra_links are set
+ self.assertCountEqual(simple_task.extra_links, [
+ 'BigQuery Console #1', 'BigQuery Console #2', 'airflow', 'github',
'google'])
+
+ ti = TaskInstance(task=simple_task, execution_date=test_date)
+ ti.xcom_push('search_query', ["dummy_value_1", "dummy_value_2"])
+
+ # Test Deserialized inbuilt link #1
+ custom_inbuilt_link = simple_task.get_extra_links(test_date, "BigQuery
Console #1")
+
self.assertEqual('https://console.cloud.google.com/bigquery?j=dummy_value_1',
custom_inbuilt_link)
+
+ # Test Deserialized inbuilt link #2
+ custom_inbuilt_link = simple_task.get_extra_links(test_date, "BigQuery
Console #2")
+
self.assertEqual('https://console.cloud.google.com/bigquery?j=dummy_value_2',
custom_inbuilt_link)
+
+ # Test Deserialized link registered via Airflow Plugin
+ google_link_from_plugin = simple_task.get_extra_links(test_date,
GoogleLink.name)
+ self.assertEqual("https://www.google.com", google_link_from_plugin)
+
+ def test_complex_operator_links_defined_in_plugins(self):
+ """
+ Assert complex OperatorLinks defined via Plugins work.
+
+ This tests also depends on GoogleLink() & CustomBaseIndexOpLink()
registered as a plugin
+ in tests/plugins/test_plugin.py
+
+ The function tests that if extra operator links are registered in
plugin
+ in ``operator_extra_links`` and the same is also defined in
+ the Operator in ``BaseOperator.operator_extra_links``, it has the
correct
+ extra link.
+ """
+
+ serialized_dag = {
+ "__version": 1,
+ "dag": {
+ "default_args": {"__type": "dict", "__var": {}},
+ "params": {},
+ "_dag_id": "simple_dag",
+ "fileloc": __file__,
+ "start_date": 1564617600.0,
+ "tasks": [
+ {
+ "task_id": "simple_task",
+ "_downstream_task_ids": [],
+ "_inlets": [],
+ "_outlets": [],
+ "_operator_extra_links": [
+ # Intentionally wrong module name is added
+ # to mimic the case where the webserver wont have
access
+ # to the OperatorLink code
Review comment:
To expand on my other comment: This can only happen when the webserver
doesn't have the plugin either -- cos if a Link is defined in a plugin, then
the normal plugin loading mechanism will (I think?) load the module under the
same python module name (same key in `sys.modules`) so the import by class name
will work on both.
That said: we need to be _very_ careful about this if it ever gets accepted
from user/API input -- this sort of thing is a possible recipie for
instantiating arbitrary python classes, which could lead to Vulns.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services