This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 5f6d7d202e Migrate GitHub example DAGs to new design #22446 (#24134)
5f6d7d202e is described below
commit 5f6d7d202eaecf8619f1a1b943166d99554256c2
Author: chethanuk-plutoflume <[email protected]>
AuthorDate: Sun Jun 5 10:34:51 2022 +0100
Migrate GitHub example DAGs to new design #22446 (#24134)
---
airflow/providers/github/example_dags/__init__.py | 16 ----
.../github/example_dags/example_github.py | 101 --------------------
docs/apache-airflow-providers-github/index.rst | 2 +-
.../operators/index.rst | 12 ++-
tests/system/providers/github/example_github.py | 105 +++++++++++++++++++++
5 files changed, 114 insertions(+), 122 deletions(-)
diff --git a/airflow/providers/github/example_dags/__init__.py
b/airflow/providers/github/example_dags/__init__.py
deleted file mode 100644
index 13a83393a9..0000000000
--- a/airflow/providers/github/example_dags/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/airflow/providers/github/example_dags/example_github.py
b/airflow/providers/github/example_dags/example_github.py
deleted file mode 100644
index 642bdd43e2..0000000000
--- a/airflow/providers/github/example_dags/example_github.py
+++ /dev/null
@@ -1,101 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-import logging
-from datetime import datetime
-from typing import Any, Optional
-
-from github import GithubException
-
-from airflow import AirflowException
-from airflow.models.dag import DAG
-from airflow.providers.github.operators.github import GithubOperator
-from airflow.providers.github.sensors.github import GithubSensor,
GithubTagSensor
-
-dag = DAG(
- 'example_github_operator',
- start_date=datetime(2021, 1, 1),
- tags=['example'],
- catchup=False,
-)
-
-# [START howto_tag_sensor_github]
-
-tag_sensor = GithubTagSensor(
- task_id='example_tag_sensor',
- tag_name='v1.0',
- repository_name="apache/airflow",
- timeout=60,
- poke_interval=10,
- dag=dag,
-)
-
-
-# [END howto_tag_sensor_github]
-
-# [START howto_sensor_github]
-
-
-def tag_checker(repo: Any, tag_name: str) -> Optional[bool]:
- result = None
- try:
- if repo is not None and tag_name is not None:
- all_tags = [x.name for x in repo.get_tags()]
- result = tag_name in all_tags
-
- except GithubException as github_error: # type: ignore[misc]
- raise AirflowException(f"Failed to execute GithubSensor, error:
{str(github_error)}")
- except Exception as e:
- raise AirflowException(f"GitHub operator error: {str(e)}")
- return result
-
-
-github_sensor = GithubSensor(
- task_id='example_sensor',
- method_name="get_repo",
- method_params={'full_name_or_id': "apache/airflow"},
- result_processor=lambda repo: tag_checker(repo, 'v1.0'),
- timeout=60,
- poke_interval=10,
- dag=dag,
-)
-
-# [END howto_sensor_github]
-
-
-# [START howto_operator_list_repos_github]
-
-github_list_repos = GithubOperator(
- task_id='github_list_repos',
- github_method="get_user",
- github_method_args={},
- result_processor=lambda user: logging.info(list(user.get_repos())),
- dag=dag,
-)
-
-# [END howto_operator_list_repos_github]
-
-# [START howto_operator_list_tags_github]
-
-list_repo_tags = GithubOperator(
- task_id='list_repo_tags',
- github_method="get_repo",
- github_method_args={'full_name_or_id': 'apache/airflow'},
- result_processor=lambda repo: logging.info(list(repo.get_tags())),
- dag=dag,
-)
-
-# [END howto_operator_list_tags_github]
diff --git a/docs/apache-airflow-providers-github/index.rst
b/docs/apache-airflow-providers-github/index.rst
index 3f3d7d5478..28a3259a64 100644
--- a/docs/apache-airflow-providers-github/index.rst
+++ b/docs/apache-airflow-providers-github/index.rst
@@ -39,7 +39,7 @@ Content
:maxdepth: 1
:caption: Resources
- Example DAGs
<https://github.com/apache/airflow/tree/main/airflow/providers/github/example_dags>
+ Example DAGs
<https://github.com/apache/airflow/tree/main/tests/system/providers/github>
.. toctree::
:maxdepth: 1
diff --git a/docs/apache-airflow-providers-github/operators/index.rst
b/docs/apache-airflow-providers-github/operators/index.rst
index 7d516c3280..9526643c41 100644
--- a/docs/apache-airflow-providers-github/operators/index.rst
+++ b/docs/apache-airflow-providers-github/operators/index.rst
@@ -29,8 +29,9 @@ You can further process the result using **result_processor**
Callable as you li
An example of Listing all Repositories owned by a user,
**client.get_user().get_repos()** can be implemented as following:
-.. exampleinclude::
/../../airflow/providers/github/example_dags/example_github.py
+.. exampleinclude:: /../../tests/system/providers/github/example_github.py
:language: python
+ :dedent: 4
:start-after: [START howto_operator_list_repos_github]
:end-before: [END howto_operator_list_repos_github]
@@ -38,8 +39,9 @@ An example of Listing all Repositories owned by a user,
**client.get_user().get_
An example of Listing Tags in a Repository,
**client.get_repo(full_name_or_id='apache/airflow').get_tags()** can be
implemented as following:
-.. exampleinclude::
/../../airflow/providers/github/example_dags/example_github.py
+.. exampleinclude:: /../../tests/system/providers/github/example_github.py
:language: python
+ :dedent: 4
:start-after: [START howto_operator_list_tags_github]
:end-before: [END howto_operator_list_tags_github]
@@ -58,14 +60,16 @@ a Tag in `GitHub <https://www.github.com/>`__.
An example for tag **v1.0**:
-.. exampleinclude::
/../../airflow/providers/github/example_dags/example_github.py
+.. exampleinclude:: /../../tests/system/providers/github/example_github.py
:language: python
+ :dedent: 4
:start-after: [START howto_tag_sensor_github]
:end-before: [END howto_tag_sensor_github]
Similar Functionality can be achieved by directly using
:class:`~airflow.providers.github.sensors.GithubSensor` ,
-.. exampleinclude::
/../../airflow/providers/github/example_dags/example_github.py
+.. exampleinclude:: /../../tests/system/providers/github/example_github.py
:language: python
+ :dedent: 4
:start-after: [START howto_sensor_github]
:end-before: [END howto_sensor_github]
diff --git a/tests/system/providers/github/example_github.py
b/tests/system/providers/github/example_github.py
new file mode 100644
index 0000000000..deb897a037
--- /dev/null
+++ b/tests/system/providers/github/example_github.py
@@ -0,0 +1,105 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+import os
+from datetime import datetime
+from typing import Any, Optional
+
+from github import GithubException
+
+from airflow import AirflowException
+from airflow.models.dag import DAG
+from airflow.providers.github.operators.github import GithubOperator
+from airflow.providers.github.sensors.github import GithubSensor,
GithubTagSensor
+
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = "example_github_operator"
+
+
+with DAG(
+ DAG_ID,
+ start_date=datetime(2021, 1, 1),
+ tags=['example'],
+ catchup=False,
+) as dag:
+
+ # [START howto_tag_sensor_github]
+
+ tag_sensor = GithubTagSensor(
+ task_id='example_tag_sensor',
+ tag_name='v1.0',
+ repository_name="apache/airflow",
+ timeout=60,
+ poke_interval=10,
+ )
+
+ # [END howto_tag_sensor_github]
+
+ # [START howto_sensor_github]
+
+ def tag_checker(repo: Any, tag_name: str) -> Optional[bool]:
+ result = None
+ try:
+ if repo is not None and tag_name is not None:
+ all_tags = [x.name for x in repo.get_tags()]
+ result = tag_name in all_tags
+
+ except GithubException as github_error: # type: ignore[misc]
+ raise AirflowException(f"Failed to execute GithubSensor, error:
{str(github_error)}")
+ except Exception as e:
+ raise AirflowException(f"GitHub operator error: {str(e)}")
+ return result
+
+ github_sensor = GithubSensor(
+ task_id='example_sensor',
+ method_name="get_repo",
+ method_params={'full_name_or_id': "apache/airflow"},
+ result_processor=lambda repo: tag_checker(repo, 'v1.0'),
+ timeout=60,
+ poke_interval=10,
+ )
+
+ # [END howto_sensor_github]
+
+ # [START howto_operator_list_repos_github]
+
+ github_list_repos = GithubOperator(
+ task_id='github_list_repos',
+ github_method="get_user",
+ github_method_args={},
+ result_processor=lambda user: logging.info(list(user.get_repos())),
+ )
+
+ # [END howto_operator_list_repos_github]
+
+ # [START howto_operator_list_tags_github]
+
+ list_repo_tags = GithubOperator(
+ task_id='list_repo_tags',
+ github_method="get_repo",
+ github_method_args={'full_name_or_id': 'apache/airflow'},
+ result_processor=lambda repo: logging.info(list(repo.get_tags())),
+ )
+
+ # [END howto_operator_list_tags_github]
+
+
+from tests.system.utils import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see:
tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)
