This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 100ea9d1fc AIP-47 - Migrate hive DAGs to new design #22439 (#24204)
100ea9d1fc is described below
commit 100ea9d1fc6831b1ea6e7d33f38c0da5ec9c5fc4
Author: chethanuk-plutoflume <[email protected]>
AuthorDate: Sun Jun 5 10:19:05 2022 +0100
AIP-47 - Migrate hive DAGs to new design #22439 (#24204)
---
docs/apache-airflow-providers-apache-hive/index.rst | 2 +-
.../apache-airflow-providers-apache-hive/operators.rst | 3 ++-
.../system/providers/apache}/__init__.py | 0
.../system/providers/apache/hive}/__init__.py | 0
.../providers/apache/hive}/example_twitter_README.md | 0
.../providers/apache/hive}/example_twitter_dag.py | 18 +++++++++++++++++-
6 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/docs/apache-airflow-providers-apache-hive/index.rst
b/docs/apache-airflow-providers-apache-hive/index.rst
index b33295d44d..be62eca213 100644
--- a/docs/apache-airflow-providers-apache-hive/index.rst
+++ b/docs/apache-airflow-providers-apache-hive/index.rst
@@ -39,7 +39,7 @@ Content
:maxdepth: 1
:caption: Resources
- Example DAGs
<https://github.com/apache/airflow/tree/main/airflow/providers/apache/hive/example_dags>
+ Example DAGs
<https://github.com/apache/airflow/tree/main/tests/system/providers/apache/hive>
PyPI Repository
<https://pypi.org/project/apache-airflow-providers-apache-hive/>
Installing from sources <installing-providers-from-sources>
diff --git a/docs/apache-airflow-providers-apache-hive/operators.rst
b/docs/apache-airflow-providers-apache-hive/operators.rst
index 4952c8092c..7a92cba9f2 100644
--- a/docs/apache-airflow-providers-apache-hive/operators.rst
+++ b/docs/apache-airflow-providers-apache-hive/operators.rst
@@ -27,8 +27,9 @@ HiveOperator
This operator executes hql code or hive script in a specific Hive database.
-.. exampleinclude::
/../../airflow/providers/apache/hive/example_dags/example_twitter_dag.py
+.. exampleinclude::
/../../tests/system/providers/apache/hive/example_twitter_dag.py
:language: python
+ :dedent: 4
:start-after: [START create_hive]
:end-before: [END create_hive]
diff --git a/airflow/providers/apache/hive/example_dags/__init__.py
b/tests/system/providers/apache/__init__.py
similarity index 100%
copy from airflow/providers/apache/hive/example_dags/__init__.py
copy to tests/system/providers/apache/__init__.py
diff --git a/airflow/providers/apache/hive/example_dags/__init__.py
b/tests/system/providers/apache/hive/__init__.py
similarity index 100%
rename from airflow/providers/apache/hive/example_dags/__init__.py
rename to tests/system/providers/apache/hive/__init__.py
diff --git
a/airflow/providers/apache/hive/example_dags/example_twitter_README.md
b/tests/system/providers/apache/hive/example_twitter_README.md
similarity index 100%
rename from airflow/providers/apache/hive/example_dags/example_twitter_README.md
rename to tests/system/providers/apache/hive/example_twitter_README.md
diff --git a/airflow/providers/apache/hive/example_dags/example_twitter_dag.py
b/tests/system/providers/apache/hive/example_twitter_dag.py
similarity index 91%
rename from airflow/providers/apache/hive/example_dags/example_twitter_dag.py
rename to tests/system/providers/apache/hive/example_twitter_dag.py
index 1110803089..9e36d3a7c1 100644
--- a/airflow/providers/apache/hive/example_dags/example_twitter_dag.py
+++ b/tests/system/providers/apache/hive/example_twitter_dag.py
@@ -26,6 +26,8 @@
"""
This is an example dag for managing twitter data.
"""
+
+import os
from datetime import date, datetime, timedelta
from airflow import DAG
@@ -33,6 +35,9 @@ from airflow.decorators import task
from airflow.operators.bash import BashOperator
from airflow.providers.apache.hive.operators.hive import HiveOperator
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = "example_twitter_dag"
+
@task
def fetch_tweets():
@@ -68,7 +73,7 @@ def transfer_to_db():
with DAG(
- dag_id='example_twitter_dag',
+ dag_id=DAG_ID,
default_args={
'owner': 'Ekhtiar',
'retries': 1,
@@ -145,3 +150,14 @@ with DAG(
)
analyze >> load_to_hdfs >> load_to_hive >> hive_to_mysql
+
+ from tests.system.utils.watcher import watcher
+
+ # This test needs watcher in order to properly mark success/failure
+ # when "tearDown" task with trigger rule is part of the DAG
+ list(dag.tasks) >> watcher()
+
+from tests.system.utils import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see:
tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)
