Repository: incubator-airflow Updated Branches: refs/heads/master 69ccf8432 -> 3475faf6b
[AIRFLOW-2302] Add missing operators and hooks Many operators and hooks are not listed in the config, and therefore not picked up by the docks generator. Closes #3201 from Fokko/airflow-2302-add-missing- docs Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/3475faf6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/3475faf6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/3475faf6 Branch: refs/heads/master Commit: 3475faf6ba9efc4ff9a08648a13f3f290ff671d7 Parents: 69ccf84 Author: Fokko Driesprong <fokkodriespr...@godatadriven.com> Authored: Wed Apr 11 11:55:29 2018 +0200 Committer: Fokko Driesprong <fokkodriespr...@godatadriven.com> Committed: Wed Apr 11 11:55:29 2018 +0200 ---------------------------------------------------------------------- .github/PULL_REQUEST_TEMPLATE.md | 7 +++++ docs/code.rst | 53 +++++++++++++++++++++++++++++------ docs/integration.rst | 41 +++++++++++++-------------- docs/kubernetes.rst | 3 +- 4 files changed, 73 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/3475faf6/.github/PULL_REQUEST_TEMPLATE.md ---------------------------------------------------------------------- diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 72cbb76..6000d0e 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -23,4 +23,11 @@ Make sure you have checked _all_ steps below. 5. Body wraps at 72 characters 6. Body explains "what" and "why", not "how" + +### Documentation +- [ ] In case of new functionality, my PR adds documentation that describes how to use it. + - When adding new operators/hooks/sensors, the autoclass documentation generation needs to be added. + + +### Code Quality - [ ] Passes `git diff upstream/master -u -- "*.py" | flake8 --diff` http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/3475faf6/docs/code.rst ---------------------------------------------------------------------- diff --git a/docs/code.rst b/docs/code.rst index 34ecec9..a30d117 100644 --- a/docs/code.rst +++ b/docs/code.rst @@ -80,6 +80,7 @@ Operators .. autoclass:: airflow.operators.python_operator.ShortCircuitOperator .. autoclass:: airflow.operators.http_operator.SimpleHttpOperator .. autoclass:: airflow.operators.slack_operator.SlackAPIOperator +.. autoclass:: airflow.operators.slack_operator.SlackAPIPostOperator .. autoclass:: airflow.operators.sqlite_operator.SqliteOperator .. autoclass:: airflow.operators.subdag_operator.SubDagOperator .. autoclass:: airflow.operators.dagrun_operator.TriggerDagRunOperator @@ -130,6 +131,7 @@ Operators .. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcSparkOperator .. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcHadoopOperator .. autoclass:: airflow.contrib.operators.dataproc_operator.DataProcPySparkOperator +.. autoclass:: airflow.contrib.operators.dataproc_operator.DataprocWorkflowTemplateBaseOperator .. autoclass:: airflow.contrib.operators.dataproc_operator.DataprocWorkflowTemplateInstantiateOperator .. autoclass:: airflow.contrib.operators.dataproc_operator.DataprocWorkflowTemplateInstantiateInlineOperator .. autoclass:: airflow.contrib.operators.datastore_export_operator.DatastoreExportOperator @@ -148,9 +150,11 @@ Operators .. autoclass:: airflow.contrib.operators.gcs_operator.GoogleCloudStorageCreateBucketOperator .. autoclass:: airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator .. autoclass:: airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageToGoogleCloudStorageOperator +.. autoclass:: airflow.contrib.operators.gcs_to_s3.GoogleCloudStorageToS3Operator .. autoclass:: airflow.contrib.operators.hipchat_operator.HipChatAPIOperator .. autoclass:: airflow.contrib.operators.hipchat_operator.HipChatAPISendRoomNotificationOperator .. autoclass:: airflow.contrib.operators.hive_to_dynamodb.HiveToDynamoDBTransferOperator +.. autoclass:: airflow.contrib.operators.jenkins_job_trigger_operator.JenkinsJobTriggerOperator .. autoclass:: airflow.contrib.operators.jira_operator.JiraOperator .. autoclass:: airflow.contrib.operators.kubernetes_pod_operator.KubernetesPodOperator .. autoclass:: airflow.contrib.operators.mlengine_operator.MLEngineBatchPredictionOperator @@ -167,13 +171,14 @@ Operators .. autoclass:: airflow.contrib.operators.qubole_operator.QuboleOperator .. autoclass:: airflow.contrib.operators.s3_list_operator.S3ListOperator .. autoclass:: airflow.contrib.operators.sftp_operator.SFTPOperator +.. autoclass:: airflow.contrib.operators.slack_webhook_operator.SlackWebhookOperator +.. autoclass:: airflow.contrib.operators.snowflake_operator.SnowflakeOperator .. autoclass:: airflow.contrib.operators.spark_jdbc_operator.SparkJDBCOperator .. autoclass:: airflow.contrib.operators.spark_sql_operator.SparkSqlOperator .. autoclass:: airflow.contrib.operators.spark_submit_operator.SparkSubmitOperator .. autoclass:: airflow.contrib.operators.sqoop_operator.SqoopOperator .. autoclass:: airflow.contrib.operators.ssh_operator.SSHOperator .. autoclass:: airflow.contrib.operators.vertica_operator.VerticaOperator -.. autoclass:: airflow.contrib.operators.spark_submit_operator.SparkSubmitOperator .. autoclass:: airflow.contrib.operators.vertica_to_hive.VerticaToHiveTransfer Sensors @@ -319,28 +324,60 @@ interface when possible and acting as building blocks for operators. .. autoclass:: airflow.hooks.http_hook.HttpHook .. autoclass:: airflow.hooks.druid_hook.DruidDbApiHook .. autoclass:: airflow.hooks.druid_hook.DruidHook +.. autoclass:: airflow.hooks.hdfs_hook.HDFSHook .. autoclass:: airflow.hooks.jdbc_hook.JdbcHook .. autoclass:: airflow.hooks.mssql_hook.MsSqlHook .. autoclass:: airflow.hooks.mysql_hook.MySqlHook +.. autoclass:: airflow.hooks.oracle_hook.OracleHook +.. autoclass:: airflow.hooks.pig_hook.PigCliHook .. autoclass:: airflow.hooks.postgres_hook.PostgresHook .. autoclass:: airflow.hooks.presto_hook.PrestoHook .. autoclass:: airflow.hooks.S3_hook.S3Hook +.. autoclass:: airflow.hooks.samba_hook.SambaHook +.. autoclass:: airflow.hooks.slack_hook.SlackHook .. autoclass:: airflow.hooks.sqlite_hook.SqliteHook .. autoclass:: airflow.hooks.webhdfs_hook.WebHDFSHook +.. autoclass:: airflow.hooks.zendesk_hook.ZendeskHook Community contributed hooks ''''''''''''''''''''''''''' -.. autoclass:: airflow.contrib.hooks.redshift_hook.RedshiftHook +.. autoclass:: airflow.contrib.hooks.aws_dynamodb_hook.AwsDynamoDBHook +.. autoclass:: airflow.contrib.hooks.aws_hook.AwsHook +.. autoclass:: airflow.contrib.hooks.aws_labmda_hook.AwsLambdaHook .. autoclass:: airflow.contrib.hooks.bigquery_hook.BigQueryHook -.. autoclass:: airflow.contrib.hooks.vertica_hook.VerticaHook -.. autoclass:: airflow.contrib.hooks.ftp_hook.FTPHook -.. autoclass:: airflow.contrib.hooks.ssh_hook.SSHHook -.. autoclass:: airflow.contrib.hooks.sftp_hook.SFTPHook .. autoclass:: airflow.contrib.hooks.cloudant_hook.CloudantHook -.. autoclass:: airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook -.. autoclass:: airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook +.. autoclass:: airflow.contrib.hooks.databricks_hook.DatabricksHook +.. autoclass:: airflow.contrib.hooks.datadog_hook.DatadogHook +.. autoclass:: airflow.contrib.hooks.datastore_hook.DatastoreHook .. autoclass:: airflow.contrib.hooks.discord_webhook_hook.DiscordWebhookHook +.. autoclass:: airflow.contrib.hooks.emr_hook.EmrHook +.. autoclass:: airflow.contrib.hooks.fs_hook.FSHook +.. autoclass:: airflow.contrib.hooks.ftp_Hook.FTPHook +.. autoclass:: airflow.contrib.hooks.ftp_Hook.FTPSHook +.. autoclass:: airflow.contrib.hooks.gcp_api_base_hook.GoogleCloudBaseHook +.. autoclass:: airflow.contrib.hooks.gcp_dataflow_hook.DataFlowHook +.. autoclass:: airflow.contrib.hooks.gcp_dataproc_hook.DataProcHook +.. autoclass:: airflow.contrib.hooks.gcp_mleengine_hook.MLEngineHook +.. autoclass:: airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook +.. autoclass:: airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook +.. autoclass:: airflow.contrib.hooks.jenkins_hook.JenkinsHook +.. autoclass:: airflow.contrib.hooks.jira_hook.JiraHook +.. autoclass:: airflow.contrib.hooks.qubole_hook.QuboleHook +.. autoclass:: airflow.contrib.hooks.redis_hook.RedisHook +.. autoclass:: airflow.contrib.hooks.redshift_hook.RedshiftHook +.. autoclass:: airflow.contrib.hooks.salesfore_hook.SalesforceHook +.. autoclass:: airflow.contrib.hooks.sftp_hook.SFTPHook +.. autoclass:: airflow.contrib.hooks.slack_webhook_hook.SlackWebhookHook +.. autoclass:: airflow.contrib.hooks.snowflake_hook.SnowflakeHook +.. autoclass:: airflow.contrib.hooks.spark_jdbc_hook.SparkJDBCHook +.. autoclass:: airflow.contrib.hooks.spark_sql_hook.SparkSqlHook +.. autoclass:: airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook +.. autoclass:: airflow.contrib.hooks.sqoop_hook.SqoopHook +.. autoclass:: airflow.contrib.hooks.ssh_hook.SSHHook +.. autoclass:: airflow.contrib.hooks.vertica_hook.VerticaHook +.. autoclass:: airflow.contrib.hooks.spark_jdbc_hook.SparkJDBCHook +.. autoclass:: airflow.contrib.hooks.wasb_hook.WasbHook Executors --------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/3475faf6/docs/integration.rst ---------------------------------------------------------------------- diff --git a/docs/integration.rst b/docs/integration.rst index 00d1ba9..d4e8b05 100644 --- a/docs/integration.rst +++ b/docs/integration.rst @@ -132,7 +132,6 @@ Follow the steps below to enable Azure Blob Storage logging. # Rename DEFAULT_LOGGING_CONFIG to LOGGING CONFIG LOGGING_CONFIG = ... - #. Make sure a Azure Blob Storage (Wasb) connection hook has been defined in Airflow. The hook should have read and write access to the Azure Blob Storage bucket defined above in ``REMOTE_BASE_LOG_FOLDER``. @@ -151,7 +150,7 @@ Follow the steps below to enable Azure Blob Storage logging. .. _AWS: AWS: Amazon Web Services ------------------------ +------------------------ Airflow has extensive support for Amazon Web Services. But note that the Hooks, Sensors and Operators are in the contrib section. @@ -167,34 +166,34 @@ AWS EMR .. _EmrAddStepsOperator: EmrAddStepsOperator -"""""""" +""""""""""""""""""" .. autoclass:: airflow.contrib.operators.emr_add_steps_operator.EmrAddStepsOperator .. _EmrCreateJobFlowOperator: EmrCreateJobFlowOperator -"""""""" +"""""""""""""""""""""""" .. autoclass:: airflow.contrib.operators.emr_create_job_flow_operator.EmrCreateJobFlowOperator .. _EmrTerminateJobFlowOperator: EmrTerminateJobFlowOperator -"""""""" +""""""""""""""""""""""""""" .. autoclass:: airflow.contrib.operators.emr_terminate_job_flow_operator.EmrTerminateJobFlowOperator .. _EmrHook: EmrHook -"""""""" +""""""" .. autoclass:: airflow.contrib.hooks.emr_hook.EmrHook AWS S3 -''''''' +'''''' - :ref:`S3ListOperator` : Lists the files matching a key prefix from a S3 location. - :ref:`S3FileTransformOperator` : Copies data from a source S3 location to a temporary location on the local filesystem. @@ -211,14 +210,14 @@ S3ListOperator .. _S3FileTransformOperator: S3FileTransformOperator -"""""""""""""""""""""""" +""""""""""""""""""""""" .. autoclass:: airflow.operators.s3_file_transform_operator.S3FileTransformOperator .. _S3ToHiveTransfer: S3ToHiveTransfer -""""""""""""""""" +"""""""""""""""" .. autoclass:: airflow.operators.s3_to_hive_operator.S3ToHiveTransfer @@ -231,33 +230,33 @@ S3Hook AWS EC2 Container Service -'''''''''''''''''''''''''' +''''''''''''''''''''''''' - :ref:`ECSOperator` : Execute a task on AWS EC2 Container Service. .. _ECSOperator: ECSOperator -"""""""""""" +""""""""""" .. autoclass:: airflow.contrib.operators.ecs_operator.ECSOperator AWS Batch Service -'''''''''''''''''''''''''' +''''''''''''''''' - :ref:`AWSBatchOperator` : Execute a task on AWS Batch Service. .. _AWSBatchOperator: AWSBatchOperator -"""""""""""" +"""""""""""""""" .. autoclass:: airflow.contrib.operators.awsbatch_operator.AWSBatchOperator AWS RedShift -''''''''''''' +'''''''''''' - :ref:`AwsRedshiftClusterSensor` : Waits for a Redshift cluster to reach a specific status. - :ref:`RedshiftHook` : Interact with AWS Redshift, using the boto3 library. @@ -266,21 +265,21 @@ AWS RedShift .. _AwsRedshiftClusterSensor: AwsRedshiftClusterSensor -""""""""""""""""""""""""" +"""""""""""""""""""""""" .. autoclass:: airflow.contrib.sensors.aws_redshift_cluster_sensor.AwsRedshiftClusterSensor .. _RedshiftHook: RedshiftHook -""""""""""""" +"""""""""""" .. autoclass:: airflow.contrib.hooks.redshift_hook.RedshiftHook .. _RedshiftToS3Transfer: RedshiftToS3Transfer -""""""""""""""""""""" +"""""""""""""""""""" .. autoclass:: airflow.operators.redshift_to_s3_operator.RedshiftToS3Transfer @@ -679,7 +678,7 @@ Cloud ML Engine Operators .. _MLEngineBatchPredictionOperator: MLEngineBatchPredictionOperator -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.mlengine_operator.MLEngineBatchPredictionOperator :members: @@ -687,7 +686,7 @@ MLEngineBatchPredictionOperator .. _MLEngineModelOperator: MLEngineModelOperator -^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.mlengine_operator.MLEngineModelOperator :members: @@ -695,7 +694,7 @@ MLEngineModelOperator .. _MLEngineTrainingOperator: MLEngineTrainingOperator -^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: airflow.contrib.operators.mlengine_operator.MLEngineTrainingOperator :members: @@ -714,7 +713,7 @@ Cloud ML Engine Hook .. _MLEngineHook: MLEngineHook -^^^^^^^^^^^ +^^^^^^^^^^^^ .. autoclass:: airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook :members: http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/3475faf6/docs/kubernetes.rst ---------------------------------------------------------------------- diff --git a/docs/kubernetes.rst b/docs/kubernetes.rst index 21d8501..3305a1e 100644 --- a/docs/kubernetes.rst +++ b/docs/kubernetes.rst @@ -1,5 +1,5 @@ Kubernetes Operator -========= +=================== @@ -27,4 +27,3 @@ Variable Description ``@labels`` Labels are an important element of launching kubernetes pods, as it tells kubernetes what pods a service can route to. For example, if you launch 5 postgres pods with the label {'postgres':'foo'} and create a postgres service with the same label, kubernetes will know that any time that service is queried, it can pick any of those 5 postgres instances as the endpoint for that service. ``@name`` name of the task you want to run, will be used to generate a pod id ================================= ==================================== -
