This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new c7788a6 Add imap_attachment_to_s3 example dag and system test (#8669)
c7788a6 is described below
commit c7788a6894cb79c22153434dd9b977393b8236be
Author: Felix Uellendall <[email protected]>
AuthorDate: Sun May 10 03:06:35 2020 +0200
Add imap_attachment_to_s3 example dag and system test (#8669)
---
.../example_dags/example_imap_attachment_to_s3.py | 53 ++++++++++++++++
.../operator/amazon/aws/imap_attachment_to_s3.rst | 70 ++++++++++++++++++++++
docs/operators-and-hooks-ref.rst | 4 +-
.../operators/test_imap_attachment_to_s3_system.py | 42 +++++++++++++
4 files changed, 167 insertions(+), 2 deletions(-)
diff --git
a/airflow/providers/amazon/aws/example_dags/example_imap_attachment_to_s3.py
b/airflow/providers/amazon/aws/example_dags/example_imap_attachment_to_s3.py
new file mode 100644
index 0000000..7a0d86c
--- /dev/null
+++ b/airflow/providers/amazon/aws/example_dags/example_imap_attachment_to_s3.py
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+This is an example dag for using `ImapAttachmentToS3Operator` to transfer an
email attachment via IMAP
+protocol from a mail server to S3 Bucket.
+"""
+
+from os import getenv
+
+from airflow import DAG
+from airflow.providers.amazon.aws.operators.imap_attachment_to_s3 import
ImapAttachmentToS3Operator
+from airflow.utils.dates import days_ago
+
+# [START howto_operator_imap_attachment_to_s3_env_variables]
+IMAP_ATTACHMENT_NAME = getenv("IMAP_ATTACHMENT_NAME", "test.txt")
+IMAP_MAIL_FOLDER = getenv("IMAP_MAIL_FOLDER", "INBOX")
+IMAP_MAIL_FILTER = getenv("IMAP_MAIL_FILTER", "All")
+S3_DESTINATION_KEY = getenv("S3_DESTINATION_KEY", "s3://bucket/key.json")
+# [END howto_operator_imap_attachment_to_s3_env_variables]
+
+default_args = {"start_date": days_ago(1)}
+
+with DAG(
+ dag_id="example_imap_attachment_to_s3",
+ default_args=default_args,
+ schedule_interval=None,
+ tags=['example']
+) as dag:
+ # [START howto_operator_imap_attachment_to_s3_task_1]
+ task_transfer_imap_attachment_to_s3 = ImapAttachmentToS3Operator(
+ imap_attachment_name=IMAP_ATTACHMENT_NAME,
+ s3_key=S3_DESTINATION_KEY,
+ imap_mail_folder=IMAP_MAIL_FOLDER,
+ imap_mail_filter=IMAP_MAIL_FILTER,
+ task_id='transfer_imap_attachment_to_s3',
+ dag=dag
+ )
+ # [END howto_operator_imap_attachment_to_s3_task_1]
diff --git a/docs/howto/operator/amazon/aws/imap_attachment_to_s3.rst
b/docs/howto/operator/amazon/aws/imap_attachment_to_s3.rst
new file mode 100644
index 0000000..94cfea3
--- /dev/null
+++ b/docs/howto/operator/amazon/aws/imap_attachment_to_s3.rst
@@ -0,0 +1,70 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+
+.. _howto/operator:ImapAttachmentToS3Operator:
+
+Imap Attachment To S3 Operator
+==============================
+
+.. contents::
+ :depth: 1
+ :local:
+
+Overview
+--------
+
+The ``ImapAttachmentToS3Operator`` can transfer an email attachment via IMAP
+protocol from a mail server to S3 Bucket.
+
+An example dag ``example_imap_attachment_to_s3.py`` is provided which showcase
the
+:class:`~airflow.providers.amazon.aws.operators.imap_attachment_to_s3.ImapAttachmentToS3Operator`
+in action.
+
+example_imap_attachment_to_s3.py
+--------------------------------
+
+Purpose
+"""""""
+This is an example dag for using ``ImapAttachmentToS3Operator`` to transfer an
email attachment via IMAP
+protocol from a mail server to S3 Bucket.
+
+Environment variables
+"""""""""""""""""""""
+
+These examples rely on the following variables, which can be passed via OS
environment variables.
+
+.. exampleinclude::
../../../../../airflow/providers/amazon/aws/example_dags/example_imap_attachment_to_s3.py
+ :language: python
+ :start-after: [START howto_operator_imap_attachment_to_s3_env_variables]
+ :end-before: [END howto_operator_imap_attachment_to_s3_env_variables]
+
+Transfer Mail Attachments via IMAP to S3
+""""""""""""""""""""""""""""""""""""""""
+
+.. exampleinclude::
../../../../../airflow/providers/amazon/aws/example_dags/example_imap_attachment_to_s3.py
+ :language: python
+ :start-after: [START howto_operator_imap_attachment_to_s3_task_1]
+ :end-before: [END howto_operator_imap_attachment_to_s3_task_1]
+
+Reference
+---------
+
+For further information, look at:
+
+* `IMAP Library Documentation
<https://docs.python.org/3.6/library/imaplib.html>`__
+* `AWS boto3 Library Documentation for S3
<https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html>`__
diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst
index 36fd657..b6f8abc 100644
--- a/docs/operators-and-hooks-ref.rst
+++ b/docs/operators-and-hooks-ref.rst
@@ -558,7 +558,7 @@ These integrations allow you to copy data from/to Amazon
Web Services.
* - `Internet Message Access Protocol (IMAP)
<https://tools.ietf.org/html/rfc3501>`__
- `Amazon Simple Storage Service (S3) <https://aws.amazon.com/s3/>`__
- -
+ - :doc:`How to use <howto/operator/amazon/aws/imap_attachment_to_s3>`
- :mod:`airflow.providers.amazon.aws.operators.imap_attachment_to_s3`
* - `MongoDB <https://www.mongodb.com/what-is-mongodb>`__
@@ -1584,7 +1584,7 @@ These integrations allow you to copy data.
* - `Internet Message Access Protocol (IMAP)
<https://tools.ietf.org/html/rfc3501>`__
- `Amazon Simple Storage Service (S3) <https://aws.amazon.com/s3/>`__
- -
+ - :doc:`How to use <howto/operator/amazon/aws/imap_attachment_to_s3>`
- :mod:`airflow.providers.amazon.aws.operators.imap_attachment_to_s3`
* - `SSH File Transfer Protocol (SFTP)
<https://tools.ietf.org/wg/secsh/draft-ietf-secsh-filexfer/>`__
diff --git
a/tests/providers/amazon/aws/operators/test_imap_attachment_to_s3_system.py
b/tests/providers/amazon/aws/operators/test_imap_attachment_to_s3_system.py
new file mode 100644
index 0000000..51b26f6
--- /dev/null
+++ b/tests/providers/amazon/aws/operators/test_imap_attachment_to_s3_system.py
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import pytest
+
+from airflow.providers.amazon.aws.example_dags.example_imap_attachment_to_s3
import S3_DESTINATION_KEY
+from airflow.providers.amazon.aws.hooks.s3 import S3Hook
+from tests.test_utils.amazon_system_helpers import (
+ AWS_DAG_FOLDER, AmazonSystemTest, provide_aws_context,
provide_aws_s3_bucket,
+)
+
+BUCKET, _ = S3Hook.parse_s3_url(S3_DESTINATION_KEY)
+
+
[email protected]
+def provide_s3_bucket():
+ with provide_aws_s3_bucket(BUCKET):
+ yield
+
+
[email protected]("mysql", "postgres")
[email protected]("imap")
+class TestImapAttachmentToS3ExampleDags(AmazonSystemTest):
+
+ @pytest.mark.usefixtures("provide_s3_bucket")
+ @provide_aws_context()
+ def test_run_example_dag_imap_attachment_to_s3(self):
+ self.run_dag('example_imap_attachment_to_s3', AWS_DAG_FOLDER)
