This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 43f334f Move docker decorator example dag to docker provider (#18739)
43f334f is described below
commit 43f334f4bdedbb39f72cb28585e9500a506480e1
Author: Ephraim Anierobi <[email protected]>
AuthorDate: Wed Oct 6 19:07:17 2021 +0100
Move docker decorator example dag to docker provider (#18739)
This example dag errors out during startup when we set
AIRFLOW__CORE__EXAMPLE_DAGS=True and docker
provider is not installed.
---
....py => tutorial_taskflow_api_etl_virtualenv.py} | 28 ++--------------------
.../tutorial_taskflow_api_etl_docker_virtualenv.py | 14 +++++++----
docs/apache-airflow/tutorial_taskflow_api.rst | 4 ++--
3 files changed, 14 insertions(+), 32 deletions(-)
diff --git
a/airflow/example_dags/tutorial_taskflow_api_etl_docker_virtualenv.py
b/airflow/example_dags/tutorial_taskflow_api_etl_virtualenv.py
similarity index 78%
copy from airflow/example_dags/tutorial_taskflow_api_etl_docker_virtualenv.py
copy to airflow/example_dags/tutorial_taskflow_api_etl_virtualenv.py
index c89ea9b..09aefcb 100644
--- a/airflow/example_dags/tutorial_taskflow_api_etl_docker_virtualenv.py
+++ b/airflow/example_dags/tutorial_taskflow_api_etl_virtualenv.py
@@ -17,29 +17,19 @@
# under the License.
-# [START tutorial]
-# [START import_module]
from datetime import datetime
from airflow.decorators import dag, task
-# [END import_module]
-
-# [START instantiate_dag]
@dag(schedule_interval=None, start_date=datetime(2021, 1, 1), catchup=False,
tags=['example'])
def tutorial_taskflow_api_etl_virtualenv():
"""
- ### TaskFlow API Tutorial Documentation
+ ### TaskFlow API example using virtualenv
This is a simple ETL data pipeline example which demonstrates the use of
the TaskFlow API using three simple tasks for Extract, Transform, and Load.
- Documentation that goes along with the Airflow TaskFlow API tutorial is
- located
-
[here](https://airflow.apache.org/docs/apache-airflow/stable/tutorial_taskflow_api.html)
"""
- # [END instantiate_dag]
- # [START extract_virtualenv]
@task.virtualenv(
use_dill=True,
system_site_packages=False,
@@ -59,10 +49,7 @@ def tutorial_taskflow_api_etl_virtualenv():
order_data_dict = json.loads(data_string)
return order_data_dict
- # [END extract_virtualenv]
-
- # [START transform_docker]
- @task.docker(image='python:3.9-slim-buster', multiple_outputs=True)
+ @task(multiple_outputs=True)
def transform(order_data_dict: dict):
"""
#### Transform task
@@ -76,9 +63,6 @@ def tutorial_taskflow_api_etl_virtualenv():
return {"total_order_value": total_order_value}
- # [END transform_docker]
-
- # [START load]
@task()
def load(total_order_value: float):
"""
@@ -89,17 +73,9 @@ def tutorial_taskflow_api_etl_virtualenv():
print(f"Total order value is: {total_order_value:.2f}")
- # [END load]
-
- # [START main_flow]
order_data = extract()
order_summary = transform(order_data)
load(order_summary["total_order_value"])
- # [END main_flow]
-# [START dag_invocation]
tutorial_etl_dag = tutorial_taskflow_api_etl_virtualenv()
-# [END dag_invocation]
-
-# [END tutorial]
diff --git
a/airflow/example_dags/tutorial_taskflow_api_etl_docker_virtualenv.py
b/airflow/providers/docker/example_dags/tutorial_taskflow_api_etl_docker_virtualenv.py
similarity index 88%
rename from airflow/example_dags/tutorial_taskflow_api_etl_docker_virtualenv.py
rename to
airflow/providers/docker/example_dags/tutorial_taskflow_api_etl_docker_virtualenv.py
index c89ea9b..1961c30 100644
--- a/airflow/example_dags/tutorial_taskflow_api_etl_docker_virtualenv.py
+++
b/airflow/providers/docker/example_dags/tutorial_taskflow_api_etl_docker_virtualenv.py
@@ -28,7 +28,7 @@ from airflow.decorators import dag, task
# [START instantiate_dag]
@dag(schedule_interval=None, start_date=datetime(2021, 1, 1), catchup=False,
tags=['example'])
-def tutorial_taskflow_api_etl_virtualenv():
+def tutorial_taskflow_api_etl_docker_virtualenv():
"""
### TaskFlow API Tutorial Documentation
This is a simple ETL data pipeline example which demonstrates the use of
@@ -98,8 +98,14 @@ def tutorial_taskflow_api_etl_virtualenv():
# [END main_flow]
-# [START dag_invocation]
-tutorial_etl_dag = tutorial_taskflow_api_etl_virtualenv()
-# [END dag_invocation]
+# The try/except here is because Airflow versions less than 2.2.0 doesn't
support
+# @task.docker decorator and we use this dag in CI test. Thus, in order not to
+# break the CI test, we added this try/except here.
+try:
+ # [START dag_invocation]
+ tutorial_etl_dag = tutorial_taskflow_api_etl_docker_virtualenv()
+ # [END dag_invocation]
+except AttributeError:
+ pass
# [END tutorial]
diff --git a/docs/apache-airflow/tutorial_taskflow_api.rst
b/docs/apache-airflow/tutorial_taskflow_api.rst
index ea5579e..ce15e4a 100644
--- a/docs/apache-airflow/tutorial_taskflow_api.rst
+++ b/docs/apache-airflow/tutorial_taskflow_api.rst
@@ -175,7 +175,7 @@ image must have a working Python installed and take in a
bash command as the ``c
Below is an example of using the ``@task.docker`` decorator to run a python
task.
-.. exampleinclude::
/../../airflow/example_dags/tutorial_taskflow_api_etl_docker_virtualenv.py
+.. exampleinclude::
/../../airflow/providers/docker/example_dags/tutorial_taskflow_api_etl_docker_virtualenv.py
:language: python
:dedent: 4
:start-after: [START transform_docker]
@@ -199,7 +199,7 @@ environment on the same machine, you can use the
``@task.virtualenv`` decorator
decorator will allow you to create a new virtualenv with custom libraries and
even a different
Python version to run your function.
-.. exampleinclude::
/../../airflow/example_dags/tutorial_taskflow_api_etl_docker_virtualenv.py
+.. exampleinclude::
/../../airflow/providers/docker/example_dags/tutorial_taskflow_api_etl_docker_virtualenv.py
:language: python
:dedent: 4
:start-after: [START extract_virtualenv]
