This is an automated email from the ASF dual-hosted git repository. kaxilnaik pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v1-10-test by this push: new 5239b4c [AIRFLOW-7066] Use sphinx syntax in concepts.rst (#7729) 5239b4c is described below commit 5239b4cdc46bca32539a5d0bf903f4464c8713d4 Author: Jiajie Zhong <zhongjiajie...@hotmail.com> AuthorDate: Sun Mar 15 20:08:32 2020 +0800 [AIRFLOW-7066] Use sphinx syntax in concepts.rst (#7729) (cherry picked from commit 0740dafa0a971b9dc05c6a1b3f034b869c827aa8) --- .../example_latest_only_with_trigger.py | 3 + airflow/example_dags/example_subdag_operator.py | 2 + airflow/example_dags/subdags/subdag.py | 2 + docs/concepts.rst | 87 +++------------------- 4 files changed, 19 insertions(+), 75 deletions(-) diff --git a/airflow/example_dags/example_latest_only_with_trigger.py b/airflow/example_dags/example_latest_only_with_trigger.py index 59b33d6..77fe89e 100644 --- a/airflow/example_dags/example_latest_only_with_trigger.py +++ b/airflow/example_dags/example_latest_only_with_trigger.py @@ -19,6 +19,8 @@ """ Example LatestOnlyOperator and TriggerRule interactions """ + +# [START example] import datetime as dt from airflow.models import DAG @@ -42,3 +44,4 @@ task4 = DummyOperator(task_id='task4', dag=dag, trigger_rule=TriggerRule.ALL_DON latest_only >> task1 >> [task3, task4] task2 >> [task3, task4] +# [END example] diff --git a/airflow/example_dags/example_subdag_operator.py b/airflow/example_dags/example_subdag_operator.py index c3a2b19..d68e60c 100644 --- a/airflow/example_dags/example_subdag_operator.py +++ b/airflow/example_dags/example_subdag_operator.py @@ -19,6 +19,7 @@ """Example DAG demonstrating the usage of the SubDagOperator.""" +# [START example_subdag_operator] from airflow.example_dags.subdags.subdag import subdag from airflow.models import DAG from airflow.operators.dummy_operator import DummyOperator @@ -67,3 +68,4 @@ end = DummyOperator( ) start >> section_1 >> some_other_task >> section_2 >> end +# [END example_subdag_operator] diff --git a/airflow/example_dags/subdags/subdag.py b/airflow/example_dags/subdags/subdag.py index 6a67c7d..ff7fee5 100644 --- a/airflow/example_dags/subdags/subdag.py +++ b/airflow/example_dags/subdags/subdag.py @@ -17,6 +17,7 @@ # specific language governing permissions and limitations # under the License. +# [START subdag] from airflow.models import DAG from airflow.operators.dummy_operator import DummyOperator @@ -36,3 +37,4 @@ def subdag(parent_dag_name, child_dag_name, args): ) return dag_subdag +# [END subdag] diff --git a/docs/concepts.rst b/docs/concepts.rst index edd6e5d..068321e 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -778,54 +778,17 @@ Note that SubDAG operators should contain a factory method that returns a DAG object. This will prevent the SubDAG from being treated like a separate DAG in the main UI. For example: -.. code:: python - - #dags/subdag.py - from airflow.models import DAG - from airflow.operators.dummy_operator import DummyOperator - - - # Dag is returned by a factory method - def sub_dag(parent_dag_name, child_dag_name, start_date, schedule_interval): - dag = DAG( - '%s.%s' % (parent_dag_name, child_dag_name), - schedule_interval=schedule_interval, - start_date=start_date, - ) - - dummy_operator = DummyOperator( - task_id='dummy_task', - dag=dag, - ) - - return dag +.. exampleinclude:: ../airflow/example_dags/subdags/subdag.py + :language: python + :start-after: [START subdag] + :end-before: [END subdag] This SubDAG can then be referenced in your main DAG file: -.. code:: python - - # main_dag.py - from datetime import datetime, timedelta - from airflow.models import DAG - from airflow.operators.subdag_operator import SubDagOperator - from dags.subdag import sub_dag - - - PARENT_DAG_NAME = 'parent_dag' - CHILD_DAG_NAME = 'child_dag' - - main_dag = DAG( - dag_id=PARENT_DAG_NAME, - schedule_interval=timedelta(hours=1), - start_date=datetime(2016, 1, 1) - ) - - sub_dag = SubDagOperator( - subdag=sub_dag(PARENT_DAG_NAME, CHILD_DAG_NAME, main_dag.start_date, - main_dag.schedule_interval), - task_id=CHILD_DAG_NAME, - dag=main_dag, - ) +.. exampleinclude:: ../airflow/example_dags/example_subdag_operator.py + :language: python + :start-after: [START example_subdag_operator] + :end-before: [END example_subdag_operator] You can zoom into a SubDagOperator from the graph view of the main DAG to show the tasks contained within the SubDAG: @@ -992,36 +955,10 @@ right now is not between its ``execution_time`` and the next scheduled For example, consider the following DAG: -.. code:: python - - #dags/latest_only_with_trigger.py - import datetime as dt - - from airflow.models import DAG - from airflow.operators.dummy_operator import DummyOperator - from airflow.operators.latest_only_operator import LatestOnlyOperator - from airflow.utils.trigger_rule import TriggerRule - - - dag = DAG( - dag_id='latest_only_with_trigger', - schedule_interval=dt.timedelta(hours=1), - start_date=dt.datetime(2019, 2, 28), - ) - - latest_only = LatestOnlyOperator(task_id='latest_only', dag=dag) - - task1 = DummyOperator(task_id='task1', dag=dag) - task1.set_upstream(latest_only) - - task2 = DummyOperator(task_id='task2', dag=dag) - - task3 = DummyOperator(task_id='task3', dag=dag) - task3.set_upstream([task1, task2]) - - task4 = DummyOperator(task_id='task4', dag=dag, - trigger_rule=TriggerRule.ALL_DONE) - task4.set_upstream([task1, task2]) +.. exampleinclude:: ../airflow/example_dags/example_latest_only_with_trigger.py + :language: python + :start-after: [START example] + :end-before: [END example] In the case of this DAG, the ``latest_only`` task will show up as skipped for all runs except the latest run. ``task1`` is directly downstream of