Repository: incubator-airflow Updated Branches: refs/heads/master 426b6a65f -> 322ec9609
[AIRFLOW-1459] Fixed broken integration .rst formatting Closes #2481 from yk5/docs Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/322ec960 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/322ec960 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/322ec960 Branch: refs/heads/master Commit: 322ec9609d977efc3771899431cfe0c52bcbf173 Parents: 426b6a6 Author: Younghee Kwon <[email protected]> Authored: Thu Jul 27 09:41:45 2017 -0700 Committer: Chris Riccomini <[email protected]> Committed: Thu Jul 27 09:42:07 2017 -0700 ---------------------------------------------------------------------- docs/integration.rst | 97 ++++++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/322ec960/docs/integration.rst ---------------------------------------------------------------------- diff --git a/docs/integration.rst b/docs/integration.rst index c31d964..30150ac 100644 --- a/docs/integration.rst +++ b/docs/integration.rst @@ -3,8 +3,10 @@ Integration - :ref:`Azure` - :ref:`AWS` +- :ref:`Databricks` - :ref:`GCP` + .. _Azure: Azure: Microsoft Azure @@ -54,26 +56,33 @@ WasbHook .. autoclass:: airflow.contrib.hooks.wasb_hook.WasbHook + + .. _AWS: AWS: Amazon Webservices ----------------------- ---- +To be filled. + + .. _Databricks: Databricks --------------------------- +---------- + `Databricks <https://databricks.com/>`_ has contributed an Airflow operator which enables submitting runs to the Databricks platform. Internally the operator talks to the ``api/2.0/jobs/runs/submit`` `endpoint <https://docs.databricks.com/api/latest/jobs.html#runs-submit>`_. DatabricksSubmitRunOperator -'''''''''''''''''''''''''''' +''''''''''''''''''''''''''' .. autoclass:: airflow.contrib.operators.databricks_operator.DatabricksSubmitRunOperator + + .. _GCP: GCP: Google Cloud Platform @@ -87,7 +96,7 @@ BigQuery '''''''' BigQuery Operators -^^^^^^^^^^^^^^^^^^ +"""""""""""""""""" - :ref:`BigQueryCheckOperator` : Performs checks against a SQL query that will return a single row with different values. - :ref:`BigQueryValueCheckOperator` : Performs a simple value check using SQL code. @@ -100,48 +109,48 @@ BigQuery Operators .. _BigQueryCheckOperator: BigQueryCheckOperator -""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.bigquery_check_operator.BigQueryCheckOperator .. _BigQueryValueCheckOperator: BigQueryValueCheckOperator -"""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.bigquery_check_operator.BigQueryValueCheckOperator .. _BigQueryIntervalCheckOperator: BigQueryIntervalCheckOperator -""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.bigquery_check_operator.BigQueryIntervalCheckOperator .. _BigQueryOperator: BigQueryOperator -"""""""""""""""" +^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.bigquery_operator.BigQueryOperator .. _BigQueryToBigQueryOperator: BigQueryToBigQueryOperator -"""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.bigquery_to_bigquery.BigQueryToBigQueryOperator .. _BigQueryToCloudStorageOperator: BigQueryToCloudStorageOperator -"""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.bigquery_to_gcs.BigQueryToCloudStorageOperator BigQueryHook -^^^^^^^^^^^^ +"""""""""""" .. autoclass:: airflow.contrib.hooks.bigquery_hook.BigQueryHook :members: @@ -151,14 +160,15 @@ Cloud DataFlow '''''''''''''' DataFlow Operators -^^^^^^^^^^^^^^^^^^ +"""""""""""""""""" -- :ref:`DataFlowJavaOperator` : +- :ref:`DataFlowJavaOperator` : launching Cloud Dataflow jobs written in Java. +- :ref:`DataFlowPythonOperator` : launching Cloud Dataflow jobs written in python. .. _DataFlowJavaOperator: DataFlowJavaOperator -"""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.dataflow_operator.DataFlowJavaOperator @@ -196,8 +206,16 @@ DataFlowJavaOperator }, dag=dag) +.. _DataFlowPythonOperator: + +DataFlowPythonOperator +^^^^^^^^^^^^^^^^^^^^^^ + +.. autoclass:: airflow.contrib.operators.dataflow_operator.DataFlowPythonOperator + + DataFlowHook -^^^^^^^^^^^^ +"""""""""""" .. autoclass:: airflow.contrib.hooks.gcp_dataflow_hook.DataFlowHook :members: @@ -208,7 +226,7 @@ Cloud DataProc '''''''''''''' DataProc Operators -^^^^^^^^^^^^^^^^^^ +"""""""""""""""""" - :ref:`DataProcPigOperator` : Start a Pig query Job on a Cloud DataProc cluster. - :ref:`DataProcHiveOperator` : Start a Hive query Job on a Cloud DataProc cluster. @@ -220,35 +238,35 @@ DataProc Operators .. _DataProcPigOperator: DataProcPigOperator -""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcPigOperator .. _DataProcHiveOperator: DataProcHiveOperator -"""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcHiveOperator .. _DataProcSparkSqlOperator: DataProcSparkSqlOperator -"""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcSparkSqlOperator .. _DataProcSparkOperator: DataProcSparkOperator -""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcSparkOperator .. _DataProcHadoopOperator: DataProcHadoopOperator -"""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcHadoopOperator @@ -264,20 +282,18 @@ DataProcPySparkOperator Cloud Datastore ''''''''''''''' -Datastore Operators -^^^^^^^^^^^^^^^^^^^ - DatastoreHook -~~~~~~~~~~~~~ +""""""""""""" .. autoclass:: airflow.contrib.hooks.datastore_hook.DatastoreHook :members: + Cloud ML Engine ''''''''''''''' Cloud ML Engine Operators -^^^^^^^^^^^^^^^^^^^^^^^^^ +""""""""""""""""""""""""" - :ref:`CloudMLBatchPredictionOperator` : Start a Cloud ML Engine batch prediction job. - :ref:`CloudMLModelOperator` : Manages a Cloud ML Engine model. @@ -287,7 +303,7 @@ Cloud ML Engine Operators .. _CloudMLBatchPredictionOperator: CloudMLBatchPredictionOperator -"""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.cloudml_operator.CloudMLBatchPredictionOperator :members: @@ -295,7 +311,7 @@ CloudMLBatchPredictionOperator .. _CloudMLModelOperator: CloudMLModelOperator -"""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.cloudml_operator.CloudMLModelOperator :members: @@ -303,7 +319,7 @@ CloudMLModelOperator .. _CloudMLTrainingOperator: CloudMLTrainingOperator -""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.cloudml_operator.CloudMLTrainingOperator :members: @@ -311,16 +327,28 @@ CloudMLTrainingOperator .. _CloudMLVersionOperator: CloudMLVersionOperator -""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.cloudml_operator.CloudMLVersionOperator :members: +Cloud ML Engine Hook +"""""""""""""""""""" + +.. _CloudMLHook: + +CloudMLHook +^^^^^^^^^^^ + +.. autoclass:: airflow.contrib.hooks.gcp_cloudml_hook.CloudMLHook + :members: + + Cloud Storage ''''''''''''' Storage Operators -^^^^^^^^^^^^^^^^^ +""""""""""""""""" - :ref:`GoogleCloudStorageDownloadOperator` : Downloads a file from Google Cloud Storage. - :ref:`GoogleCloudStorageToBigQueryOperator` : Loads files from Google cloud storage into BigQuery. @@ -328,7 +356,7 @@ Storage Operators .. _GoogleCloudStorageDownloadOperator: GoogleCloudStorageDownloadOperator -"""""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator :members: @@ -336,15 +364,14 @@ GoogleCloudStorageDownloadOperator .. _GoogleCloudStorageToBigQueryOperator: GoogleCloudStorageToBigQueryOperator -"""""""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator :members: GoogleCloudStorageHook -^^^^^^^^^^^^^^^^^^^^^^ +"""""""""""""""""""""" .. autoclass:: airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook :members: -
