This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 57b7ba16a3 Doc changes: Added Transfers section in Azure provider docs
(#32241)
57b7ba16a3 is described below
commit 57b7ba16a3d860268f03cd2619e5d029c7994013
Author: Akash Sharma <[email protected]>
AuthorDate: Thu Jun 29 19:31:25 2023 +0530
Doc changes: Added Transfers section in Azure provider docs (#32241)
---
airflow/providers/microsoft/azure/provider.yaml | 7 +--
.../index.rst | 1 +
.../{operators => transfer}/azure_blob_to_gcs.rst | 0
.../transfer/index.rst | 28 ++++++++++++
.../{operators => transfer}/local_to_adls.rst | 0
.../local_to_wasb.rst} | 50 +++++++++-------------
.../{operators => transfer}/sftp_to_wasb.rst | 0
.../microsoft/azure/example_local_to_wasb.py | 12 +++++-
8 files changed, 63 insertions(+), 35 deletions(-)
diff --git a/airflow/providers/microsoft/azure/provider.yaml
b/airflow/providers/microsoft/azure/provider.yaml
index d57a948372..c255da9315 100644
--- a/airflow/providers/microsoft/azure/provider.yaml
+++ b/airflow/providers/microsoft/azure/provider.yaml
@@ -230,21 +230,22 @@ triggers:
transfers:
- source-integration-name: Local
target-integration-name: Microsoft Azure Data Lake Storage
- how-to-guide:
/docs/apache-airflow-providers-microsoft-azure/operators/local_to_adls.rst
+ how-to-guide:
/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_adls.rst
python-module: airflow.providers.microsoft.azure.transfers.local_to_adls
- source-integration-name: Oracle
target-integration-name: Microsoft Azure Data Lake Storage
python-module:
airflow.providers.microsoft.azure.transfers.oracle_to_azure_data_lake
- source-integration-name: Local
target-integration-name: Microsoft Azure Blob Storage
+ how-to-guide:
/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst
python-module: airflow.providers.microsoft.azure.transfers.local_to_wasb
- source-integration-name: Microsoft Azure Blob Storage
target-integration-name: Google Cloud Storage (GCS)
- how-to-guide:
/docs/apache-airflow-providers-microsoft-azure/operators/azure_blob_to_gcs.rst
+ how-to-guide:
/docs/apache-airflow-providers-microsoft-azure/transfer/azure_blob_to_gcs.rst
python-module:
airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs
- source-integration-name: SSH File Transfer Protocol (SFTP)
target-integration-name: Microsoft Azure Blob Storage
- how-to-guide:
/docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst
+ how-to-guide:
/docs/apache-airflow-providers-microsoft-azure/transfer/sftp_to_wasb.rst
python-module: airflow.providers.microsoft.azure.transfers.sftp_to_wasb
diff --git a/docs/apache-airflow-providers-microsoft-azure/index.rst
b/docs/apache-airflow-providers-microsoft-azure/index.rst
index 5820216b13..dfc0bf56e1 100644
--- a/docs/apache-airflow-providers-microsoft-azure/index.rst
+++ b/docs/apache-airflow-providers-microsoft-azure/index.rst
@@ -36,6 +36,7 @@
Connection types <connections/index>
Operators <operators/index>
+ Transfers <transfer/index>
Secrets backends <secrets-backends/azure-key-vault>
Logging for Tasks <logging/index>
diff --git
a/docs/apache-airflow-providers-microsoft-azure/operators/azure_blob_to_gcs.rst
b/docs/apache-airflow-providers-microsoft-azure/transfer/azure_blob_to_gcs.rst
similarity index 100%
rename from
docs/apache-airflow-providers-microsoft-azure/operators/azure_blob_to_gcs.rst
rename to
docs/apache-airflow-providers-microsoft-azure/transfer/azure_blob_to_gcs.rst
diff --git a/docs/apache-airflow-providers-microsoft-azure/transfer/index.rst
b/docs/apache-airflow-providers-microsoft-azure/transfer/index.rst
new file mode 100644
index 0000000000..846e1057a2
--- /dev/null
+++ b/docs/apache-airflow-providers-microsoft-azure/transfer/index.rst
@@ -0,0 +1,28 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+
+
+Azure Transfer Operators
+=========================
+
+
+.. toctree::
+ :maxdepth: 1
+ :glob:
+
+ *
diff --git
a/docs/apache-airflow-providers-microsoft-azure/operators/local_to_adls.rst
b/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_adls.rst
similarity index 100%
rename from
docs/apache-airflow-providers-microsoft-azure/operators/local_to_adls.rst
rename to
docs/apache-airflow-providers-microsoft-azure/transfer/local_to_adls.rst
diff --git
a/docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst
b/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst
similarity index 55%
copy from
docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst
copy to docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst
index a79500de6a..970a629834 100644
--- a/docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst
+++ b/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst
@@ -1,4 +1,3 @@
-
.. Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@@ -17,45 +16,36 @@
under the License.
-
-SFTP to Azure Blob Storage Transfer Operator
-============================================
+Upload data from Local Filesystem to Azure Blob Storage
+=======================================================
The Blob service stores text and binary data as objects in the cloud.
The Blob service offers the following three resources: the storage account,
containers, and blobs.
Within your storage account, containers provide a way to organize sets of
blobs.
For more information about the service visit `Azure Blob Storage API
documentation
<https://docs.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api>`_.
+This page shows how to upload data from local filesystem to Azure Blob Storage.
+
-Before you begin
-^^^^^^^^^^^^^^^^
-Before using Blob Storage within Airflow you need to authenticate your account
with Token, Login and Password.
-Please follow Azure
-`instructions
<https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal>`_
-to do it.
+.. _howto/operator:LocalFilesystemToWasbOperator:
-See following example.
-Set values for these fields:
+LocalFilesystemToWasbOperator
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-.. code-block::
+:class:`~airflow.providers.microsoft.azure.transfers.local_to_wasb.LocalFilesystemToWasbOperator`
allows you to
+upload data from local filesystem to Azure Blob Storage.
- SFTP Conn Id: sftp_default
- WASB Conn Id: wasb_default
-.. contents::
- :depth: 1
- :local:
+Below is an example of using this operator to upload a file to Azure Blob
Storage.
-.. _howto/operator:SFTPToWasbOperator:
+.. exampleinclude::
/../../tests/system/providers/microsoft/azure/example_local_to_wasb.py
+ :language: python
+ :dedent: 0
+ :start-after: [START howto_operator_local_to_wasb]
+ :end-before: [END howto_operator_local_to_wasb]
-Transfer Data from SFTP Source Path to Blob Storage
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Operator transfers data from SFTP Source Path to specified container in Azure
Blob Storage
-To get information about jobs within a Azure Blob Storage use:
-:class:`~airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPToWasbOperator`
-Example usage:
+Reference
+---------
-.. exampleinclude::
/../../tests/system/providers/microsoft/azure/example_sftp_to_wasb.py
- :language: python
- :dedent: 4
- :start-after: [START how_to_sftp_to_wasb]
- :end-before: [END how_to_sftp_to_wasb]
+For further information, look at:
+
+* `Azure Blob Storage Documentation
<https://learn.microsoft.com/en-us/azure/storage/blobs/>`__
diff --git
a/docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst
b/docs/apache-airflow-providers-microsoft-azure/transfer/sftp_to_wasb.rst
similarity index 100%
rename from
docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst
rename to
docs/apache-airflow-providers-microsoft-azure/transfer/sftp_to_wasb.rst
diff --git a/tests/system/providers/microsoft/azure/example_local_to_wasb.py
b/tests/system/providers/microsoft/azure/example_local_to_wasb.py
index cab99713f9..7fff09c135 100644
--- a/tests/system/providers/microsoft/azure/example_local_to_wasb.py
+++ b/tests/system/providers/microsoft/azure/example_local_to_wasb.py
@@ -30,15 +30,23 @@ from
airflow.providers.microsoft.azure.transfers.local_to_wasb import LocalFiles
PATH_TO_UPLOAD_FILE = os.environ.get("AZURE_PATH_TO_UPLOAD_FILE",
"example-text.txt")
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
DAG_ID = "example_local_to_wasb"
+AZURE_CONTAINER_NAME = os.environ.get("AZURE_CONTAINER_NAME", "mycontainer")
+AZURE_BLOB_NAME = os.environ.get("AZURE_BLOB_NAME", "myblob")
with DAG(
DAG_ID,
schedule="@once",
start_date=datetime(2021, 1, 1),
catchup=False,
- default_args={"container_name": "mycontainer", "blob_name": "myblob"},
) as dag:
- upload = LocalFilesystemToWasbOperator(task_id="upload_file",
file_path=PATH_TO_UPLOAD_FILE)
+ # [START howto_operator_local_to_wasb]
+ upload = LocalFilesystemToWasbOperator(
+ task_id="upload_file",
+ file_path=PATH_TO_UPLOAD_FILE,
+ container_name=AZURE_CONTAINER_NAME,
+ blob_name=AZURE_BLOB_NAME,
+ )
+ # [END howto_operator_local_to_wasb]
delete = WasbDeleteBlobOperator(task_id="delete_file")
upload >> delete
