[AIRFLOW-2493] Mark template_fields of all Operators in the API document as "templated"
Make all the "template_fields" (jinjia template) of all Operators marked as "templated" in the API document. Closes #3386 from imroc/AIRFLOW-2493 Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/dc78b919 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/dc78b919 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/dc78b919 Branch: refs/heads/master Commit: dc78b9196723ca6724185231ccd6f5bbe8edcaf3 Parents: b755d35 Author: roc <[email protected]> Authored: Sun May 20 14:03:58 2018 +0100 Committer: Kaxil Naik <[email protected]> Committed: Sun May 20 14:03:58 2018 +0100 ---------------------------------------------------------------------- airflow/contrib/operators/awsbatch_operator.py | 3 +- airflow/contrib/operators/bigquery_get_data.py | 6 +- airflow/contrib/operators/bigquery_operator.py | 24 +++--- .../operators/bigquery_table_delete_operator.py | 2 +- .../contrib/operators/bigquery_to_bigquery.py | 7 +- airflow/contrib/operators/bigquery_to_gcs.py | 6 +- .../contrib/operators/databricks_operator.py | 3 +- airflow/contrib/operators/dataproc_operator.py | 82 +++++++++++--------- .../operators/discord_webhook_operator.py | 4 +- airflow/contrib/operators/ecs_operator.py | 2 +- .../contrib/operators/emr_add_steps_operator.py | 4 +- .../operators/emr_create_job_flow_operator.py | 3 +- .../emr_terminate_job_flow_operator.py | 2 +- airflow/contrib/operators/file_to_gcs.py | 6 +- airflow/contrib/operators/file_to_wasb.py | 6 +- .../contrib/operators/gcs_download_operator.py | 8 +- airflow/contrib/operators/gcs_list_operator.py | 7 +- airflow/contrib/operators/gcs_operator.py | 8 +- airflow/contrib/operators/gcs_to_bq.py | 10 +-- airflow/contrib/operators/gcs_to_gcs.py | 10 +-- airflow/contrib/operators/gcs_to_s3.py | 9 ++- airflow/contrib/operators/hipchat_operator.py | 4 +- airflow/contrib/operators/hive_to_dynamodb.py | 2 +- .../operators/jenkins_job_trigger_operator.py | 2 +- airflow/contrib/operators/jira_operator.py | 2 +- .../operators/kubernetes_pod_operator.py | 6 +- airflow/contrib/operators/mlengine_operator.py | 49 ++++++------ airflow/contrib/operators/pubsub_operator.py | 3 +- airflow/contrib/operators/s3_list_operator.py | 6 +- airflow/contrib/operators/s3_to_gcs_operator.py | 8 +- .../operators/segment_track_event_operator.py | 6 +- airflow/contrib/operators/sftp_operator.py | 4 +- airflow/contrib/operators/snowflake_operator.py | 2 +- airflow/contrib/operators/spark_sql_operator.py | 2 +- .../contrib/operators/spark_submit_operator.py | 7 +- airflow/contrib/operators/ssh_operator.py | 2 +- airflow/contrib/operators/vertica_operator.py | 2 +- airflow/contrib/operators/vertica_to_hive.py | 7 +- airflow/contrib/operators/vertica_to_mysql.py | 8 +- airflow/contrib/operators/winrm_operator.py | 2 +- airflow/operators/bash_operator.py | 2 +- airflow/operators/check_operator.py | 4 +- airflow/operators/docker_operator.py | 4 +- airflow/operators/email_operator.py | 8 +- airflow/operators/generic_transfer.py | 6 +- airflow/operators/hive_operator.py | 7 +- airflow/operators/hive_stats_operator.py | 4 +- airflow/operators/hive_to_druid.py | 6 +- airflow/operators/hive_to_mysql.py | 14 ++-- airflow/operators/hive_to_samba_operator.py | 2 +- airflow/operators/http_operator.py | 4 +- airflow/operators/jdbc_operator.py | 2 +- airflow/operators/mssql_operator.py | 3 +- airflow/operators/mssql_to_hive.py | 9 ++- airflow/operators/mysql_operator.py | 2 +- airflow/operators/mysql_to_hive.py | 6 +- airflow/operators/oracle_operator.py | 2 +- airflow/operators/pig_operator.py | 2 +- airflow/operators/postgres_operator.py | 2 +- airflow/operators/presto_to_mysql.py | 10 +-- airflow/operators/python_operator.py | 2 +- airflow/operators/s3_file_transform_operator.py | 4 +- airflow/operators/s3_to_hive_operator.py | 6 +- airflow/operators/slack_operator.py | 10 ++- airflow/operators/sqlite_operator.py | 2 +- 65 files changed, 240 insertions(+), 219 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/awsbatch_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/awsbatch_operator.py b/airflow/contrib/operators/awsbatch_operator.py index f3678e1..920fdb7 100644 --- a/airflow/contrib/operators/awsbatch_operator.py +++ b/airflow/contrib/operators/awsbatch_operator.py @@ -39,7 +39,8 @@ class AWSBatchOperator(BaseOperator): :type job_definition: str :param queue: the queue name on AWS Batch :type queue: str - :param: overrides: the same parameter that boto3 will receive on containerOverrides: + :param: overrides: the same parameter that boto3 will receive on + containerOverrides (templated): http://boto3.readthedocs.io/en/latest/reference/services/batch.html#submit_job :type: overrides: dict :param max_retries: exponential backoff retries while waiter is not merged http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/bigquery_get_data.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/bigquery_get_data.py b/airflow/contrib/operators/bigquery_get_data.py index 3f94d6c..aabfddd 100644 --- a/airflow/contrib/operators/bigquery_get_data.py +++ b/airflow/contrib/operators/bigquery_get_data.py @@ -50,12 +50,12 @@ class BigQueryGetDataOperator(BaseOperator): bigquery_conn_id='airflow-service-account' ) - :param dataset_id: The dataset ID of the requested table. + :param dataset_id: The dataset ID of the requested table. (templated) :type destination_dataset_table: string - :param table_id: The table ID of the requested table. + :param table_id: The table ID of the requested table. (templated) :type table_id: string :param max_results: The maximum number of records (rows) to be fetched - from the table. + from the table. (templated) :type max_results: string :param selected_fields: List of fields to return (comma-separated). If unspecified, all fields are returned. http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/bigquery_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/bigquery_operator.py b/airflow/contrib/operators/bigquery_operator.py index 416b0ac..04ddb6a 100644 --- a/airflow/contrib/operators/bigquery_operator.py +++ b/airflow/contrib/operators/bigquery_operator.py @@ -32,10 +32,10 @@ class BigQueryOperator(BaseOperator): :param bql: the sql code to be executed :type bql: Can receive a str representing a sql statement, a list of str (sql statements), or reference to a template file. - Template reference are recognized by str ending in '.sql' + Template reference are recognized by str ending in '.sql'. (templated) :param destination_dataset_table: A dotted (<project>.|<project>:)<dataset>.<table> that, if set, will store the results - of the query. + of the query. (templated) :type destination_dataset_table: string :param write_disposition: Specifies the action that occurs if the destination table already exists. (default: 'WRITE_EMPTY') @@ -160,11 +160,11 @@ class BigQueryCreateEmptyTableOperator(BaseOperator): Google cloud storage must be a JSON file with the schema fields in it. You can also create a table without schema. - :param project_id: The project to create the table into. + :param project_id: The project to create the table into. (templated) :type project_id: string - :param dataset_id: The dataset to create the table into. + :param dataset_id: The dataset to create the table into. (templated) :type dataset_id: string - :param table_id: The Name of the table to be created. + :param table_id: The Name of the table to be created. (templated) :type table_id: string :param schema_fields: If set, the schema field list as defined here: https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema @@ -175,7 +175,8 @@ class BigQueryCreateEmptyTableOperator(BaseOperator): {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"}] :type schema_fields: list - :param gcs_schema_object: Full path to the JSON file containing schema. For + :param gcs_schema_object: Full path to the JSON file containing + schema (templated). For example: ``gs://test-bucket/dir1/dir2/employee_schema.json`` :type gcs_schema_object: string :param time_partitioning: configure optional time partitioning fields i.e. @@ -302,14 +303,15 @@ class BigQueryCreateExternalTableOperator(BaseOperator): point the operator to a Google cloud storage object name. The object in Google cloud storage must be a JSON file with the schema fields in it. - :param bucket: The bucket to point the external table to. + :param bucket: The bucket to point the external table to. (templated) :type bucket: string - :param source_objects: List of Google cloud storage URIs to point table to. + :param source_objects: List of Google cloud storage URIs to point + table to. (templated) If source_format is 'DATASTORE_BACKUP', the list must only contain a single URI. :type object: list :param destination_project_dataset_table: The dotted (<project>.)<dataset>.<table> - BigQuery table to load data into. If <project> is not included, project will - be the project defined in the connection json. + BigQuery table to load data into (templated). If <project> is not included, + project will be the project defined in the connection json. :type destination_project_dataset_table: string :param schema_fields: If set, the schema field list as defined here: https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.load.schema @@ -322,7 +324,7 @@ class BigQueryCreateExternalTableOperator(BaseOperator): Should not be set when source_format is 'DATASTORE_BACKUP'. :type schema_fields: list :param schema_object: If set, a GCS object path pointing to a .json file that - contains the schema for the table. + contains the schema for the table. (templated) :param schema_object: string :param source_format: File format of the data. :type source_format: string http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/bigquery_table_delete_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/bigquery_table_delete_operator.py b/airflow/contrib/operators/bigquery_table_delete_operator.py index b3de07b..8751dfe 100644 --- a/airflow/contrib/operators/bigquery_table_delete_operator.py +++ b/airflow/contrib/operators/bigquery_table_delete_operator.py @@ -28,7 +28,7 @@ class BigQueryTableDeleteOperator(BaseOperator): :param deletion_dataset_table: A dotted (<project>.|<project>:)<dataset>.<table> that indicates which table - will be deleted. + will be deleted. (templated) :type deletion_dataset_table: string :param bigquery_conn_id: reference to a specific BigQuery hook. :type bigquery_conn_id: string http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/bigquery_to_bigquery.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/bigquery_to_bigquery.py b/airflow/contrib/operators/bigquery_to_bigquery.py index d0d375b..b03511a 100644 --- a/airflow/contrib/operators/bigquery_to_bigquery.py +++ b/airflow/contrib/operators/bigquery_to_bigquery.py @@ -32,11 +32,12 @@ class BigQueryToBigQueryOperator(BaseOperator): :param source_project_dataset_tables: One or more dotted (project:|project.)<dataset>.<table> BigQuery tables to use as the - source data. If <project> is not included, project will be the project defined - in the connection json. Use a list if there are multiple source tables. + source data. If <project> is not included, project will be the + project defined in the connection json. Use a list if there are multiple + source tables. (templated) :type source_project_dataset_tables: list|string :param destination_project_dataset_table: The destination BigQuery - table. Format is: (project:|project.)<dataset>.<table> + table. Format is: (project:|project.)<dataset>.<table> (templated) :type destination_project_dataset_table: string :param write_disposition: The write disposition if the table already exists. :type write_disposition: string http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/bigquery_to_gcs.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/bigquery_to_gcs.py b/airflow/contrib/operators/bigquery_to_gcs.py index e0840cf..166d05e 100644 --- a/airflow/contrib/operators/bigquery_to_gcs.py +++ b/airflow/contrib/operators/bigquery_to_gcs.py @@ -32,11 +32,11 @@ class BigQueryToCloudStorageOperator(BaseOperator): :param source_project_dataset_table: The dotted (<project>.|<project>:)<dataset>.<table> BigQuery table to use as the source - data. If <project> is not included, project will be the project defined in - the connection json. + data. If <project> is not included, project will be the project + defined in the connection json. (templated) :type source_project_dataset_table: string :param destination_cloud_storage_uris: The destination Google Cloud - Storage URI (e.g. gs://some-bucket/some-file.txt). Follows + Storage URI (e.g. gs://some-bucket/some-file.txt). (templated) Follows convention defined here: https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple :type destination_cloud_storage_uris: list http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/databricks_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/databricks_operator.py b/airflow/contrib/operators/databricks_operator.py index 2aa1ef2..8d22d4c 100644 --- a/airflow/contrib/operators/databricks_operator.py +++ b/airflow/contrib/operators/databricks_operator.py @@ -88,8 +88,7 @@ class DatabricksSubmitRunOperator(BaseOperator): (i.e. ``spark_jar_task``, ``notebook_task``..) to this operator will be merged with this json dictionary if they are provided. If there are conflicts during the merge, the named parameters will - take precedence and override the top level json keys. This field will be - templated. + take precedence and override the top level json keys. (templated) .. seealso:: For more information about templating see :ref:`jinja-templating`. http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/dataproc_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/dataproc_operator.py b/airflow/contrib/operators/dataproc_operator.py index 23fae9a..22db3e6 100644 --- a/airflow/contrib/operators/dataproc_operator.py +++ b/airflow/contrib/operators/dataproc_operator.py @@ -47,10 +47,10 @@ class DataprocClusterCreateOperator(BaseOperator): for a detailed explanation on the different parameters. Most of the configuration parameters detailed in the link are available as a parameter to this operator. - :param cluster_name: The name of the DataProc cluster to create. + :param cluster_name: The name of the DataProc cluster to create. (templated) :type cluster_name: string :param project_id: The ID of the google cloud project in which - to create the cluster + to create the cluster. (templated) :type project_id: string :param num_workers: The # of workers to spin up :type num_workers: int @@ -85,7 +85,7 @@ class DataprocClusterCreateOperator(BaseOperator): :type num_preemptible_workers: int :param labels: dict of labels to add to the cluster :type labels: dict - :param zone: The zone where the cluster will be located + :param zone: The zone where the cluster will be located. (templated) :type zone: string :param network_uri: The network uri to be used for machine communication, cannot be specified with subnetwork_uri @@ -95,7 +95,7 @@ class DataprocClusterCreateOperator(BaseOperator): :type subnetwork_uri: string :param tags: The GCE tags to add to all instances :type tags: list[string] - :param region: leave as 'global', might become relevant in the future + :param region: leave as 'global', might become relevant in the future. (templated) :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform. :type gcp_conn_id: string :param delegate_to: The account to impersonate, if any. @@ -397,12 +397,12 @@ class DataprocClusterScaleOperator(BaseOperator): For more detail on about scaling clusters have a look at the reference: https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/scaling-clusters - :param cluster_name: The name of the cluster to scale. + :param cluster_name: The name of the cluster to scale. (templated) :type cluster_name: string :param project_id: The ID of the google cloud project in which - the cluster runs + the cluster runs. (templated) :type project_id: string - :param region: The region for the dataproc cluster + :param region: The region for the dataproc cluster. (templated) :type region: string :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform. :type gcp_conn_id: string @@ -530,12 +530,12 @@ class DataprocClusterDeleteOperator(BaseOperator): Delete a cluster on Google Cloud Dataproc. The operator will wait until the cluster is destroyed. - :param cluster_name: The name of the cluster to create. + :param cluster_name: The name of the cluster to create. (templated) :type cluster_name: string :param project_id: The ID of the google cloud project in which - the cluster runs + the cluster runs. (templated) :type project_id: string - :param region: leave as 'global', might become relevant in the future + :param region: leave as 'global', might become relevant in the future. (templated) :type region: string :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform. :type gcp_conn_id: string @@ -630,17 +630,19 @@ class DataProcPigOperator(BaseOperator): For more detail on about job submission have a look at the reference: https://cloud.google.com/dataproc/reference/rest/v1/projects.regions.jobs - :param query: The query or reference to the query file (pg or pig extension). + :param query: The query or reference to the query + file (pg or pig extension). (templated) :type query: string :param query_uri: The uri of a pig script on Cloud Storage. :type query_uri: string - :param variables: Map of named parameters for the query. + :param variables: Map of named parameters for the query. (templated) :type variables: dict - :param job_name: The job name used in the DataProc cluster. This name by default - is the task_id appended with the execution data, but can be templated. The - name will always be appended with a random number to avoid name clashes. + :param job_name: The job name used in the DataProc cluster. This + name by default is the task_id appended with the execution data, but can + be templated. The name will always be appended with a random number to + avoid name clashes. (templated) :type job_name: string - :param cluster_name: The name of the DataProc cluster. + :param cluster_name: The name of the DataProc cluster. (templated) :type cluster_name: string :param dataproc_pig_properties: Map for the Pig properties. Ideal to put in default arguments @@ -791,17 +793,18 @@ class DataProcSparkSqlOperator(BaseOperator): """ Start a Spark SQL query Job on a Cloud DataProc cluster. - :param query: The query or reference to the query file (q extension). + :param query: The query or reference to the query file (q extension). (templated) :type query: string :param query_uri: The uri of a spark sql script on Cloud Storage. :type query_uri: string - :param variables: Map of named parameters for the query. + :param variables: Map of named parameters for the query. (templated) :type variables: dict - :param job_name: The job name used in the DataProc cluster. This name by default - is the task_id appended with the execution data, but can be templated. The - name will always be appended with a random number to avoid name clashes. + :param job_name: The job name used in the DataProc cluster. This + name by default is the task_id appended with the execution data, but can + be templated. The name will always be appended with a random number to + avoid name clashes. (templated) :type job_name: string - :param cluster_name: The name of the DataProc cluster. + :param cluster_name: The name of the DataProc cluster. (templated) :type cluster_name: string :param dataproc_spark_properties: Map for the Pig properties. Ideal to put in default arguments @@ -878,18 +881,19 @@ class DataProcSparkOperator(BaseOperator): :param main_class: Name of the job class. (use this or the main_jar, not both together). :type main_class: string - :param arguments: Arguments for the job. + :param arguments: Arguments for the job. (templated) :type arguments: list :param archives: List of archived files that will be unpacked in the work directory. Should be stored in Cloud Storage. :type archives: list :param files: List of files to be copied to the working directory :type files: list - :param job_name: The job name used in the DataProc cluster. This name by default - is the task_id appended with the execution data, but can be templated. The - name will always be appended with a random number to avoid name clashes. + :param job_name: The job name used in the DataProc cluster. This + name by default is the task_id appended with the execution data, but can + be templated. The name will always be appended with a random number to + avoid name clashes. (templated) :type job_name: string - :param cluster_name: The name of the DataProc cluster. + :param cluster_name: The name of the DataProc cluster. (templated) :type cluster_name: string :param dataproc_spark_properties: Map for the Pig properties. Ideal to put in default arguments @@ -968,18 +972,19 @@ class DataProcHadoopOperator(BaseOperator): :param main_class: Name of the job class. (use this or the main_jar, not both together). :type main_class: string - :param arguments: Arguments for the job. + :param arguments: Arguments for the job. (templated) :type arguments: list :param archives: List of archived files that will be unpacked in the work directory. Should be stored in Cloud Storage. :type archives: list :param files: List of files to be copied to the working directory :type files: list - :param job_name: The job name used in the DataProc cluster. This name by default - is the task_id appended with the execution data, but can be templated. The - name will always be appended with a random number to avoid name clashes. + :param job_name: The job name used in the DataProc cluster. This + name by default is the task_id appended with the execution data, but can + be templated. The name will always be appended with a random number to + avoid name clashes. (templated) :type job_name: string - :param cluster_name: The name of the DataProc cluster. + :param cluster_name: The name of the DataProc cluster. (templated) :type cluster_name: string :param dataproc_hadoop_properties: Map for the Pig properties. Ideal to put in default arguments @@ -1055,7 +1060,7 @@ class DataProcPySparkOperator(BaseOperator): :param main: [Required] The Hadoop Compatible Filesystem (HCFS) URI of the main Python file to use as the driver. Must be a .py file. :type main: string - :param arguments: Arguments for the job. + :param arguments: Arguments for the job. (templated) :type arguments: list :param archives: List of archived files that will be unpacked in the work directory. Should be stored in Cloud Storage. @@ -1065,9 +1070,10 @@ class DataProcPySparkOperator(BaseOperator): :param pyfiles: List of Python files to pass to the PySpark framework. Supported file types: .py, .egg, and .zip :type pyfiles: list - :param job_name: The job name used in the DataProc cluster. This name by default - is the task_id appended with the execution data, but can be templated. The - name will always be appended with a random number to avoid name clashes. + :param job_name: The job name used in the DataProc cluster. This + name by default is the task_id appended with the execution data, but can + be templated. The name will always be appended with a random number to + avoid name clashes. (templated) :type job_name: string :param cluster_name: The name of the DataProc cluster. :type cluster_name: string @@ -1214,7 +1220,7 @@ class DataprocWorkflowTemplateInstantiateOperator(DataprocWorkflowTemplateBaseOp Please refer to: https://cloud.google.com/dataproc/docs/reference/rest/v1beta2/projects.regions.workflowTemplates/instantiate - :param template_id: The id of the template. + :param template_id: The id of the template. (templated) :type template_id: string :param project_id: The ID of the google cloud project in which the template runs @@ -1258,7 +1264,7 @@ class DataprocWorkflowTemplateInstantiateInlineOperator( Please refer to: https://cloud.google.com/dataproc/docs/reference/rest/v1beta2/projects.regions.workflowTemplates/instantiateInline - :param template: The template contents. + :param template: The template contents. (templated) :type template: map :param project_id: The ID of the google cloud project in which the template runs http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/discord_webhook_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/discord_webhook_operator.py b/airflow/contrib/operators/discord_webhook_operator.py index a5c9236..acfba58 100644 --- a/airflow/contrib/operators/discord_webhook_operator.py +++ b/airflow/contrib/operators/discord_webhook_operator.py @@ -41,9 +41,9 @@ class DiscordWebhookOperator(SimpleHttpOperator): "webhooks/{webhook.id}/{webhook.token}" :type webhook_endpoint: str :param message: The message you want to send to your Discord channel - (max 2000 characters) + (max 2000 characters). (templated) :type message: str - :param username: Override the default username of the webhook + :param username: Override the default username of the webhook. (templated) :type username: str :param avatar_url: Override the default avatar of the webhook :type avatar_url: str http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/ecs_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/ecs_operator.py b/airflow/contrib/operators/ecs_operator.py index 104898c..1016fca 100644 --- a/airflow/contrib/operators/ecs_operator.py +++ b/airflow/contrib/operators/ecs_operator.py @@ -33,7 +33,7 @@ class ECSOperator(BaseOperator): :type task_definition: str :param cluster: the cluster name on EC2 Container Service :type cluster: str - :param: overrides: the same parameter that boto3 will receive: + :param: overrides: the same parameter that boto3 will receive (templated): http://boto3.readthedocs.org/en/latest/reference/services/ecs.html#ECS.Client.run_task :type: overrides: dict :param aws_conn_id: connection id of AWS credentials / region name. If None, http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/emr_add_steps_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/emr_add_steps_operator.py b/airflow/contrib/operators/emr_add_steps_operator.py index 5e3fae1..3d2d510 100644 --- a/airflow/contrib/operators/emr_add_steps_operator.py +++ b/airflow/contrib/operators/emr_add_steps_operator.py @@ -26,11 +26,11 @@ class EmrAddStepsOperator(BaseOperator): """ An operator that adds steps to an existing EMR job_flow. - :param job_flow_id: id of the JobFlow to add steps to + :param job_flow_id: id of the JobFlow to add steps to. (templated) :type job_flow_name: str :param aws_conn_id: aws connection to uses :type aws_conn_id: str - :param steps: boto3 style steps to be added to the jobflow + :param steps: boto3 style steps to be added to the jobflow. (templated) :type steps: list """ template_fields = ['job_flow_id', 'steps'] http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/emr_create_job_flow_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/emr_create_job_flow_operator.py b/airflow/contrib/operators/emr_create_job_flow_operator.py index f5a3370..e90c77a 100644 --- a/airflow/contrib/operators/emr_create_job_flow_operator.py +++ b/airflow/contrib/operators/emr_create_job_flow_operator.py @@ -31,7 +31,8 @@ class EmrCreateJobFlowOperator(BaseOperator): :type aws_conn_id: str :param emr_conn_id: emr connection to use :type emr_conn_id: str - :param job_flow_overrides: boto3 style arguments to override emr_connection extra + :param job_flow_overrides: boto3 style arguments to override + emr_connection extra. (templated) :type steps: dict """ template_fields = ['job_flow_overrides'] http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/emr_terminate_job_flow_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/emr_terminate_job_flow_operator.py b/airflow/contrib/operators/emr_terminate_job_flow_operator.py index 9d21092..95c3164 100644 --- a/airflow/contrib/operators/emr_terminate_job_flow_operator.py +++ b/airflow/contrib/operators/emr_terminate_job_flow_operator.py @@ -26,7 +26,7 @@ class EmrTerminateJobFlowOperator(BaseOperator): """ Operator to terminate EMR JobFlows. - :param job_flow_id: id of the JobFlow to terminate + :param job_flow_id: id of the JobFlow to terminate. (templated) :type job_flow_name: str :param aws_conn_id: aws connection to uses :type aws_conn_id: str http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/file_to_gcs.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/file_to_gcs.py b/airflow/contrib/operators/file_to_gcs.py index dfd5ac2..b65e3b3 100644 --- a/airflow/contrib/operators/file_to_gcs.py +++ b/airflow/contrib/operators/file_to_gcs.py @@ -27,11 +27,11 @@ class FileToGoogleCloudStorageOperator(BaseOperator): """ Uploads a file to Google Cloud Storage - :param src: Path to the local file + :param src: Path to the local file. (templated) :type src: string - :param dst: Destination path within the specified bucket + :param dst: Destination path within the specified bucket. (templated) :type dst: string - :param bucket: The bucket to upload to + :param bucket: The bucket to upload to. (templated) :type bucket: string :param google_cloud_storage_conn_id: The Airflow connection ID to upload with :type google_cloud_storage_conn_id: string http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/file_to_wasb.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/file_to_wasb.py b/airflow/contrib/operators/file_to_wasb.py index 4a13af5..f55c0c1 100644 --- a/airflow/contrib/operators/file_to_wasb.py +++ b/airflow/contrib/operators/file_to_wasb.py @@ -26,11 +26,11 @@ class FileToWasbOperator(BaseOperator): """ Uploads a file to Azure Blob Storage. - :param file_path: Path to the file to load. + :param file_path: Path to the file to load. (templated) :type file_path: str - :param container_name: Name of the container. + :param container_name: Name of the container. (templated) :type container_name: str - :param blob_name: Name of the blob. + :param blob_name: Name of the blob. (templated) :type blob_name: str :param wasb_conn_id: Reference to the wasb connection. :type wasb_conn_id: str http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/gcs_download_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/gcs_download_operator.py b/airflow/contrib/operators/gcs_download_operator.py index d1bbb65..7dfa96c 100644 --- a/airflow/contrib/operators/gcs_download_operator.py +++ b/airflow/contrib/operators/gcs_download_operator.py @@ -28,19 +28,19 @@ class GoogleCloudStorageDownloadOperator(BaseOperator): """ Downloads a file from Google Cloud Storage. - :param bucket: The Google cloud storage bucket where the object is. + :param bucket: The Google cloud storage bucket where the object is. (templated) :type bucket: string :param object: The name of the object to download in the Google cloud - storage bucket. + storage bucket. (templated) :type object: string :param filename: The file path on the local file system (where the - operator is being executed) that the file should be downloaded to. + operator is being executed) that the file should be downloaded to. (templated) If no filename passed, the downloaded data will not be stored on the local file system. :type filename: string :param store_to_xcom_key: If this param is set, the operator will push the contents of the downloaded file to XCom with the key set in this - parameter. If not set, the downloaded data will not be pushed to XCom. + parameter. If not set, the downloaded data will not be pushed to XCom. (templated) :type store_to_xcom_key: string :param google_cloud_storage_conn_id: The connection ID to use when connecting to Google cloud storage. http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/gcs_list_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/gcs_list_operator.py b/airflow/contrib/operators/gcs_list_operator.py index 3c40b9f..393deb9 100644 --- a/airflow/contrib/operators/gcs_list_operator.py +++ b/airflow/contrib/operators/gcs_list_operator.py @@ -29,11 +29,12 @@ class GoogleCloudStorageListOperator(BaseOperator): This operator returns a python list with the name of objects which can be used by `xcom` in the downstream task. - :param bucket: The Google cloud storage bucket to find the objects. + :param bucket: The Google cloud storage bucket to find the objects. (templated) :type bucket: string - :param prefix: Prefix string which filters objects whose name begin with this prefix + :param prefix: Prefix string which filters objects whose name begin with + this prefix. (templated) :type prefix: string - :param delimiter: The delimiter by which you want to filter the objects. + :param delimiter: The delimiter by which you want to filter the objects. (templated) For e.g to lists the CSV files from in a directory in GCS you would use delimiter='.csv'. :type delimiter: string http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/gcs_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/gcs_operator.py b/airflow/contrib/operators/gcs_operator.py index 3ce6a92..3d339d8 100644 --- a/airflow/contrib/operators/gcs_operator.py +++ b/airflow/contrib/operators/gcs_operator.py @@ -32,10 +32,10 @@ class GoogleCloudStorageCreateBucketOperator(BaseOperator): For more information, see Bucket Naming Guidelines: https://cloud.google.com/storage/docs/bucketnaming.html#requirements - :param bucket_name: The name of the bucket. + :param bucket_name: The name of the bucket. (templated) :type bucket_name: string :param storage_class: This defines how objects in the bucket are stored - and determines the SLA and the cost of storage. Values include + and determines the SLA and the cost of storage (templated). Values include - ``MULTI_REGIONAL`` - ``REGIONAL`` @@ -45,7 +45,7 @@ class GoogleCloudStorageCreateBucketOperator(BaseOperator): If this value is not specified when the bucket is created, it will default to STANDARD. :type storage_class: string - :param location: The location of the bucket. + :param location: The location of the bucket. (templated) Object data for objects in the bucket resides in physical storage within this region. Defaults to US. @@ -53,7 +53,7 @@ class GoogleCloudStorageCreateBucketOperator(BaseOperator): https://developers.google.com/storage/docs/bucket-locations :type location: string - :param project_id: The ID of the GCP Project. + :param project_id: The ID of the GCP Project. (templated) :type project_id: string :param labels: User-provided labels, in key/value pairs. :type labels: dict http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/gcs_to_bq.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/gcs_to_bq.py b/airflow/contrib/operators/gcs_to_bq.py index 84f55ed..938da4e 100644 --- a/airflow/contrib/operators/gcs_to_bq.py +++ b/airflow/contrib/operators/gcs_to_bq.py @@ -34,21 +34,21 @@ class GoogleCloudStorageToBigQueryOperator(BaseOperator): point the operator to a Google cloud storage object name. The object in Google cloud storage must be a JSON file with the schema fields in it. - :param bucket: The bucket to load from. + :param bucket: The bucket to load from. (templated) :type bucket: string - :param source_objects: List of Google cloud storage URIs to load from. + :param source_objects: List of Google cloud storage URIs to load from. (templated) If source_format is 'DATASTORE_BACKUP', the list must only contain a single URI. :type object: list :param destination_project_dataset_table: The dotted (<project>.)<dataset>.<table> - BigQuery table to load data into. If <project> is not included, project will - be the project defined in the connection json. + BigQuery table to load data into. If <project> is not included, + project will be the project defined in the connection json. (templated) :type destination_project_dataset_table: string :param schema_fields: If set, the schema field list as defined here: https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load Should not be set when source_format is 'DATASTORE_BACKUP'. :type schema_fields: list :param schema_object: If set, a GCS object path pointing to a .json file that - contains the schema for the table. + contains the schema for the table. (templated) :param schema_object: string :param source_format: File format to export. :type source_format: string http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/gcs_to_gcs.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/gcs_to_gcs.py b/airflow/contrib/operators/gcs_to_gcs.py index 466e631..e6d327b 100644 --- a/airflow/contrib/operators/gcs_to_gcs.py +++ b/airflow/contrib/operators/gcs_to_gcs.py @@ -26,10 +26,11 @@ class GoogleCloudStorageToGoogleCloudStorageOperator(BaseOperator): """ Copies objects from a bucket to another, with renaming if requested. - :param source_bucket: The source Google cloud storage bucket where the object is. + :param source_bucket: The source Google cloud storage bucket where the + object is. (templated) :type source_bucket: string :param source_object: The source name of the object to copy in the Google cloud - storage bucket. + storage bucket. (templated) If wildcards are used in this argument: You can use only one wildcard for objects (filenames) within your bucket. The wildcard can appear inside the object name or at the @@ -37,11 +38,10 @@ class GoogleCloudStorageToGoogleCloudStorageOperator(BaseOperator): unsupported. :type source_object: string :param destination_bucket: The destination Google cloud storage bucket - where the object should be. + where the object should be. (templated) :type destination_bucket: string :param destination_object: The destination name of the object in the - destination Google cloud - storage bucket. + destination Google cloud storage bucket. (templated) If a wildcard is supplied in the source_object argument, this is the prefix that will be prepended to the final destination objects' paths. Note that the source path's part before the wildcard will be removed; http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/gcs_to_s3.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/gcs_to_s3.py b/airflow/contrib/operators/gcs_to_s3.py index 5a73b01..00c5e14 100644 --- a/airflow/contrib/operators/gcs_to_s3.py +++ b/airflow/contrib/operators/gcs_to_s3.py @@ -27,11 +27,12 @@ class GoogleCloudStorageToS3Operator(GoogleCloudStorageListOperator): """ Synchronizes a Google Cloud Storage bucket with an S3 bucket. - :param bucket: The Google Cloud Storage bucket to find the objects. + :param bucket: The Google Cloud Storage bucket to find the objects. (templated) :type bucket: string - :param prefix: Prefix string which filters objects whose name begin with this prefix + :param prefix: Prefix string which filters objects whose name begin with + this prefix. (templated) :type prefix: string - :param delimiter: The delimiter by which you want to filter the objects. + :param delimiter: The delimiter by which you want to filter the objects. (templated) For e.g to lists the CSV files from in a directory in GCS you would use delimiter='.csv'. :type delimiter: string @@ -44,7 +45,7 @@ class GoogleCloudStorageToS3Operator(GoogleCloudStorageListOperator): :type delegate_to: string :param dest_aws_conn_id: The destination S3 connection :type dest_aws_conn_id: str - :param dest_s3_key: The base S3 key to be used to store the files + :param dest_s3_key: The base S3 key to be used to store the files. (templated) :type dest_s3_key: str """ template_fields = ('bucket', 'prefix', 'delimiter', 'dest_s3_key') http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/hipchat_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/hipchat_operator.py b/airflow/contrib/operators/hipchat_operator.py index 9bb43f1..5dd06f6 100644 --- a/airflow/contrib/operators/hipchat_operator.py +++ b/airflow/contrib/operators/hipchat_operator.py @@ -82,9 +82,9 @@ class HipChatAPISendRoomNotificationOperator(HipChatAPIOperator): Send notification to a specific HipChat room. More info: https://www.hipchat.com/docs/apiv2/method/send_room_notification - :param room_id: Room in which to send notification on HipChat + :param room_id: Room in which to send notification on HipChat. (templated) :type room_id: str - :param message: The message body + :param message: The message body. (templated) :type message: str :param frm: Label to be shown in addition to sender's name :type frm: str http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/hive_to_dynamodb.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/hive_to_dynamodb.py b/airflow/contrib/operators/hive_to_dynamodb.py index 31cca7c..2b6f88f 100644 --- a/airflow/contrib/operators/hive_to_dynamodb.py +++ b/airflow/contrib/operators/hive_to_dynamodb.py @@ -31,7 +31,7 @@ class HiveToDynamoDBTransferOperator(BaseOperator): into memory before being pushed to DynamoDB, so this operator should be used for smallish amount of data. - :param sql: SQL query to execute against the hive database + :param sql: SQL query to execute against the hive database. (templated) :type sql: str :param table_name: target DynamoDB table :type table_name: str http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/jenkins_job_trigger_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/jenkins_job_trigger_operator.py b/airflow/contrib/operators/jenkins_job_trigger_operator.py index 33d602e..ad59797 100644 --- a/airflow/contrib/operators/jenkins_job_trigger_operator.py +++ b/airflow/contrib/operators/jenkins_job_trigger_operator.py @@ -98,7 +98,7 @@ class JenkinsJobTriggerOperator(BaseOperator): :type jenkins_connection_id: string :param job_name: The name of the job to trigger :type job_name: string - :param parameters: The parameters block to provide to jenkins + :param parameters: The parameters block to provide to jenkins. (templated) :type parameters: string :param sleep_time: How long will the operator sleep between each status request for the job (min 1, default 10) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/jira_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/jira_operator.py b/airflow/contrib/operators/jira_operator.py index d18f5e5..64869e4 100644 --- a/airflow/contrib/operators/jira_operator.py +++ b/airflow/contrib/operators/jira_operator.py @@ -34,7 +34,7 @@ class JiraOperator(BaseOperator): :type jira_conn_id: str :param jira_method: method name from Jira Python SDK to be called :type jira_method: str - :param jira_method_args: required method parameters for the jira_method + :param jira_method_args: required method parameters for the jira_method. (templated) :type jira_method_args: dict :param result_processor: function to further process the response from Jira :type result_processor: function http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/kubernetes_pod_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/kubernetes_pod_operator.py b/airflow/contrib/operators/kubernetes_pod_operator.py index 31ffd92..391ec58 100644 --- a/airflow/contrib/operators/kubernetes_pod_operator.py +++ b/airflow/contrib/operators/kubernetes_pod_operator.py @@ -38,10 +38,10 @@ class KubernetesPodOperator(BaseOperator): :type image: str :param: namespace: the namespace to run within kubernetes :type: namespace: str - :param cmds: entrypoint of the container. + :param cmds: entrypoint of the container. (templated) The docker images's entrypoint is used if this is not provide. :type cmds: list of str - :param arguments: arguments of to the entrypoint. + :param arguments: arguments of to the entrypoint. (templated) The docker image's CMD is used if this is not provided. :type arguments: list of str :param volume_mounts: volumeMounts for launched pod @@ -55,7 +55,7 @@ class KubernetesPodOperator(BaseOperator): :param name: name of the task you want to run, will be used to generate a pod id :type name: str - :param env_vars: Environment variables initialized in the container + :param env_vars: Environment variables initialized in the container. (templated) :type env_vars: dict :param secrets: Kubernetes secrets to inject in the container, They can be exposed as environment vars or files in a volume. http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/mlengine_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/mlengine_operator.py b/airflow/contrib/operators/mlengine_operator.py index a6f186b..9fe966d 100644 --- a/airflow/contrib/operators/mlengine_operator.py +++ b/airflow/contrib/operators/mlengine_operator.py @@ -93,11 +93,11 @@ class MLEngineBatchPredictionOperator(BaseOperator): for further documentation on the parameters. :param project_id: The Google Cloud project name where the - prediction job is submitted. + prediction job is submitted. (templated) :type project_id: string :param job_id: A unique id for the prediction job on Google Cloud - ML Engine. + ML Engine. (templated) :type job_id: string :param data_format: The format of the input data. @@ -106,32 +106,32 @@ class MLEngineBatchPredictionOperator(BaseOperator): :type data_format: string :param input_paths: A list of GCS paths of input data for batch - prediction. Accepting wildcard operator *, but only at the end. + prediction. Accepting wildcard operator *, but only at the end. (templated) :type input_paths: list of string :param output_path: The GCS path where the prediction results are - written to. + written to. (templated) :type output_path: string :param region: The Google Compute Engine region to run the - prediction job in.: + prediction job in. (templated) :type region: string :param model_name: The Google Cloud ML Engine model to use for prediction. If version_name is not provided, the default version of this model will be used. Should not be None if version_name is provided. - Should be None if uri is provided. + Should be None if uri is provided. (templated) :type model_name: string :param version_name: The Google Cloud ML Engine model version to use for prediction. - Should be None if uri is provided. + Should be None if uri is provided. (templated) :type version_name: string :param uri: The GCS path of the saved model to use for prediction. Should be None if model_name is provided. - It should be a GCS path pointing to a tensorflow SavedModel. + It should be a GCS path pointing to a tensorflow SavedModel. (templated) :type uri: string :param max_worker_count: The maximum number of workers to be used @@ -280,7 +280,7 @@ class MLEngineModelOperator(BaseOperator): Operator for managing a Google Cloud ML Engine model. :param project_id: The Google Cloud project name to which MLEngine - model belongs. + model belongs. (templated) :type project_id: string :param model: A dictionary containing the information about the model. @@ -345,13 +345,13 @@ class MLEngineVersionOperator(BaseOperator): :type project_id: string :param model_name: The name of the Google Cloud ML Engine model that the version - belongs to. + belongs to. (templated) :type model_name: string - :param version_name: A name to use for the version being operated upon. If - not None and the `version` argument is None or does not have a value for + :param version_name: A name to use for the version being operated upon. + If not None and the `version` argument is None or does not have a value for the `name` key, then this will be populated in the payload for the - `name` key. + `name` key. (templated) :type version_name: string :param version: A dictionary containing the information about the version. @@ -359,7 +359,7 @@ class MLEngineVersionOperator(BaseOperator): information about this version such as name, and deploymentUrl. If the `operation` is `get` or `delete`, the `version` parameter should contain the `name` of the version. - If it is None, the only `operation` possible would be `list`. + If it is None, the only `operation` possible would be `list`. (templated) :type version: dict :param operation: The operation to perform. Available operations are: @@ -447,41 +447,42 @@ class MLEngineTrainingOperator(BaseOperator): Operator for launching a MLEngine training job. :param project_id: The Google Cloud project name within which MLEngine - training job should run. This field could be templated. + training job should run (templated). :type project_id: string :param job_id: A unique templated id for the submitted Google MLEngine - training job. + training job. (templated) :type job_id: string :param package_uris: A list of package locations for MLEngine training job, which should include the main training program + any additional - dependencies. + dependencies. (templated) :type package_uris: string :param training_python_module: The Python module name to run within MLEngine - training job after installing 'package_uris' packages. + training job after installing 'package_uris' packages. (templated) :type training_python_module: string :param training_args: A list of templated command line arguments to pass to - the MLEngine training program. + the MLEngine training program. (templated) :type training_args: string :param region: The Google Compute Engine region to run the MLEngine training - job in. This field could be templated. + job in (templated). :type region: string - :param scale_tier: Resource tier for MLEngine training job. + :param scale_tier: Resource tier for MLEngine training job. (templated) :type scale_tier: string - :param runtime_version: The Google Cloud ML runtime version to use for training. + :param runtime_version: The Google Cloud ML runtime version to use for + training. (templated) :type runtime_version: string - :param python_version: The version of Python used in training. + :param python_version: The version of Python used in training. (templated) :type python_version: string :param job_dir: A Google Cloud Storage path in which to store training - outputs and other data needed for training. + outputs and other data needed for training. (templated) :type job_dir: string :param gcp_conn_id: The connection ID to use when fetching connection info. http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/pubsub_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/pubsub_operator.py b/airflow/contrib/operators/pubsub_operator.py index 0822453..42586aa 100644 --- a/airflow/contrib/operators/pubsub_operator.py +++ b/airflow/contrib/operators/pubsub_operator.py @@ -408,9 +408,8 @@ class PubSubPublishOperator(BaseOperator): * 'data': a base64-encoded string * 'attributes': {'key1': 'value1', ...} Each message must contain at least a non-empty 'data' value - or an attribute dict with at least one key. See + or an attribute dict with at least one key (templated). See https://cloud.google.com/pubsub/docs/reference/rest/v1/PubsubMessage - (templated) :type messages: list :param gcp_conn_id: The connection ID to use connecting to Google Cloud Platform. http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/s3_list_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/s3_list_operator.py b/airflow/contrib/operators/s3_list_operator.py index dbb45fe..df448ad 100644 --- a/airflow/contrib/operators/s3_list_operator.py +++ b/airflow/contrib/operators/s3_list_operator.py @@ -29,12 +29,12 @@ class S3ListOperator(BaseOperator): This operator returns a python list with the name of objects which can be used by `xcom` in the downstream task. - :param bucket: The S3 bucket where to find the objects. + :param bucket: The S3 bucket where to find the objects. (templated) :type bucket: string :param prefix: Prefix string to filters the objects whose name begin with - such prefix + such prefix. (templated) :type prefix: string - :param delimiter: the delimiter marks key hierarchy. + :param delimiter: the delimiter marks key hierarchy. (templated) :type delimiter: string :param aws_conn_id: The connection ID to use when connecting to S3 storage. :type aws_conn_id: string http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/s3_to_gcs_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/s3_to_gcs_operator.py b/airflow/contrib/operators/s3_to_gcs_operator.py index 5a2004d..2898af1 100644 --- a/airflow/contrib/operators/s3_to_gcs_operator.py +++ b/airflow/contrib/operators/s3_to_gcs_operator.py @@ -32,12 +32,12 @@ class S3ToGoogleCloudStorageOperator(S3ListOperator): Synchronizes an S3 key, possibly a prefix, with a Google Cloud Storage destination path. - :param bucket: The S3 bucket where to find the objects. + :param bucket: The S3 bucket where to find the objects. (templated) :type bucket: string :param prefix: Prefix string which filters objects whose name begin with - such prefix. + such prefix. (templated) :type prefix: string - :param delimiter: the delimiter marks key hierarchy. + :param delimiter: the delimiter marks key hierarchy. (templated) :type delimiter: string :param aws_conn_id: The source S3 connection :type aws_conn_id: string @@ -45,7 +45,7 @@ class S3ToGoogleCloudStorageOperator(S3ListOperator): when connecting to Google Cloud Storage. :type dest_gcs_conn_id: string :param dest_gcs: The destination Google Cloud Storage bucket and prefix - where you want to store the files. + where you want to store the files. (templated) :type dest_gcs: string :param delegate_to: The account to impersonate, if any. For this to work, the service account making the request must have http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/segment_track_event_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/segment_track_event_operator.py b/airflow/contrib/operators/segment_track_event_operator.py index faacce8..04f6ae6 100644 --- a/airflow/contrib/operators/segment_track_event_operator.py +++ b/airflow/contrib/operators/segment_track_event_operator.py @@ -26,11 +26,11 @@ class SegmentTrackEventOperator(BaseOperator): """ Send Track Event to Segment for a specified user_id and event - :param user_id: The ID for this user in your database + :param user_id: The ID for this user in your database. (templated) :type user_id: string - :param event: The name of the event you're tracking + :param event: The name of the event you're tracking. (templated) :type event: string - :param properties: A dictionary of properties for the event. + :param properties: A dictionary of properties for the event. (templated) :type properties: dict :param segment_conn_id: The connection ID to use when connecting to Segment. :type segment_conn_id: string http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/sftp_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/sftp_operator.py b/airflow/contrib/operators/sftp_operator.py index 6ceaacb..519dc1d 100644 --- a/airflow/contrib/operators/sftp_operator.py +++ b/airflow/contrib/operators/sftp_operator.py @@ -39,9 +39,9 @@ class SFTPOperator(BaseOperator): :type ssh_conn_id: str :param remote_host: remote host to connect :type remote_host: str - :param local_filepath: local file path to get or put + :param local_filepath: local file path to get or put. (templated) :type local_filepath: str - :param remote_filepath: remote file path to get or put + :param remote_filepath: remote file path to get or put. (templated) :type remote_filepath: str :param operation: specify operation 'get' or 'put', defaults to get :type get: bool http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/snowflake_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/snowflake_operator.py b/airflow/contrib/operators/snowflake_operator.py index f59ee8b..25e9bf4 100644 --- a/airflow/contrib/operators/snowflake_operator.py +++ b/airflow/contrib/operators/snowflake_operator.py @@ -27,7 +27,7 @@ class SnowflakeOperator(BaseOperator): :param snowflake_conn_id: reference to specific snowflake connection id :type snowflake_conn_id: string - :param sql: the sql code to be executed + :param sql: the sql code to be executed. (templated) :type sql: Can receive a str representing a sql statement, a list of str (sql statements), or reference to a template file. Template reference are recognized by str ending in '.sql' http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/spark_sql_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/spark_sql_operator.py b/airflow/contrib/operators/spark_sql_operator.py index a67d939..864c137 100644 --- a/airflow/contrib/operators/spark_sql_operator.py +++ b/airflow/contrib/operators/spark_sql_operator.py @@ -26,7 +26,7 @@ class SparkSqlOperator(BaseOperator): """ Execute Spark SQL query - :param sql: The SQL query to execute + :param sql: The SQL query to execute. (templated) :type sql: str :param conf: arbitrary Spark configuration property :type conf: str (format: PROP=VALUE) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/spark_submit_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/spark_submit_operator.py b/airflow/contrib/operators/spark_submit_operator.py index 27bd61b..3c3cab0 100644 --- a/airflow/contrib/operators/spark_submit_operator.py +++ b/airflow/contrib/operators/spark_submit_operator.py @@ -29,7 +29,8 @@ class SparkSubmitOperator(BaseOperator): It requires that the "spark-submit" binary is in the PATH or the spark-home is set in the extra on the connection. - :param application: The application that submitted as a job, either jar or py file. + :param application: The application that submitted as a job, either jar or + py file. (templated) :type application: str :param conf: Arbitrary Spark configuration properties :type conf: dict @@ -49,7 +50,7 @@ class SparkSubmitOperator(BaseOperator): :param java_class: the main class of the Java application :type java_class: str :param packages: Comma-separated list of maven coordinates of jars to include on the - driver and executor classpaths + driver and executor classpaths. (templated) :type packages: str :param exclude_packages: Comma-separated list of maven coordinates of jars to exclude while resolving the dependencies provided in 'packages' @@ -71,7 +72,7 @@ class SparkSubmitOperator(BaseOperator): :type keytab: str :param principal: The name of the kerberos principal used for keytab :type principal: str - :param name: Name of the job (default airflow-spark) + :param name: Name of the job (default airflow-spark). (templated) :type name: str :param num_executors: Number of executors to launch :type num_executors: int http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/ssh_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/ssh_operator.py b/airflow/contrib/operators/ssh_operator.py index 0ee0b71..b5ab2dc 100644 --- a/airflow/contrib/operators/ssh_operator.py +++ b/airflow/contrib/operators/ssh_operator.py @@ -37,7 +37,7 @@ class SSHOperator(BaseOperator): :type ssh_conn_id: str :param remote_host: remote host to connect :type remote_host: str - :param command: command to execute on remote host + :param command: command to execute on remote host. (templated) :type command: str :param timeout: timeout (in seconds) for executing the command. :type timeout: int http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/vertica_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/vertica_operator.py b/airflow/contrib/operators/vertica_operator.py index 8629dcd..c2a6efb 100644 --- a/airflow/contrib/operators/vertica_operator.py +++ b/airflow/contrib/operators/vertica_operator.py @@ -27,7 +27,7 @@ class VerticaOperator(BaseOperator): :param vertica_conn_id: reference to a specific Vertica database :type vertica_conn_id: string - :param sql: the sql code to be executed + :param sql: the sql code to be executed. (templated) :type sql: Can receive a str representing a sql statement, a list of str (sql statements), or reference to a template file. Template reference are recognized by str ending in '.sql' http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/vertica_to_hive.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/vertica_to_hive.py b/airflow/contrib/operators/vertica_to_hive.py index c9bf823..40166fa 100644 --- a/airflow/contrib/operators/vertica_to_hive.py +++ b/airflow/contrib/operators/vertica_to_hive.py @@ -43,16 +43,17 @@ class VerticaToHiveTransfer(BaseOperator): stage the data into a temporary table before loading it into its final destination using a ``HiveOperator``. - :param sql: SQL query to execute against the Vertia database + :param sql: SQL query to execute against the Vertia database. (templated) :type sql: str :param hive_table: target Hive table, use dot notation to target a - specific database + specific database. (templated) :type hive_table: str :param create: whether to create the table if it doesn't exist :type create: bool :param recreate: whether to drop and recreate the table at every execution :type recreate: bool - :param partition: target partition as a dict of partition columns and values + :param partition: target partition as a dict of partition columns + and values. (templated) :type partition: dict :param delimiter: field delimiter in the file :type delimiter: str http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/vertica_to_mysql.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/vertica_to_mysql.py b/airflow/contrib/operators/vertica_to_mysql.py index 2e9325d..22af0ce 100644 --- a/airflow/contrib/operators/vertica_to_mysql.py +++ b/airflow/contrib/operators/vertica_to_mysql.py @@ -34,23 +34,23 @@ class VerticaToMySqlTransfer(BaseOperator): """ Moves data from Vertica to MySQL. - :param sql: SQL query to execute against the Vertica database + :param sql: SQL query to execute against the Vertica database. (templated) :type sql: str :param vertica_conn_id: source Vertica connection :type vertica_conn_id: str :param mysql_table: target MySQL table, use dot notation to target a - specific database + specific database. (templated) :type mysql_table: str :param mysql_conn_id: source mysql connection :type mysql_conn_id: str :param mysql_preoperator: sql statement to run against MySQL prior to import, typically use to truncate of delete in place of the data coming in, allowing the task to be idempotent (running the task - twice won't double load data) + twice won't double load data). (templated) :type mysql_preoperator: str :param mysql_postoperator: sql statement to run against MySQL after the import, typically used to move data from staging to production - and issue cleanup commands. + and issue cleanup commands. (templated) :type mysql_postoperator: str :param bulk_load: flag to use bulk_load option. This loads MySQL directly from a tab-delimited text file using the LOAD DATA LOCAL INFILE command. http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/contrib/operators/winrm_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/winrm_operator.py b/airflow/contrib/operators/winrm_operator.py index c426c02..fcd2328 100644 --- a/airflow/contrib/operators/winrm_operator.py +++ b/airflow/contrib/operators/winrm_operator.py @@ -34,7 +34,7 @@ class WinRMOperator(BaseOperator): :type ssh_conn_id: str :param remote_host: remote host to connect :type remote_host: str - :param command: command to execute on remote host + :param command: command to execute on remote host. (templated) :type command: str :param timeout: timeout for executing the command. :type timeout: int http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/operators/bash_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/bash_operator.py b/airflow/operators/bash_operator.py index 7db562e..53a68a7 100644 --- a/airflow/operators/bash_operator.py +++ b/airflow/operators/bash_operator.py @@ -45,7 +45,7 @@ class BashOperator(BaseOperator): Execute a Bash script, command or set of commands. :param bash_command: The command, set of commands or reference to a - bash script (must be '.sh') to be executed. + bash script (must be '.sh') to be executed. (templated) :type bash_command: string :param xcom_push: If xcom_push is True, the last line written to stdout will also be pushed to an XCom when the bash command completes. http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/operators/check_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/check_operator.py b/airflow/operators/check_operator.py index e671fda..fc9dc16 100644 --- a/airflow/operators/check_operator.py +++ b/airflow/operators/check_operator.py @@ -58,7 +58,7 @@ class CheckOperator(BaseOperator): needs to be defined. Whereas a get_db_hook is hook that gets a single record from an external source. - :param sql: the sql to be executed + :param sql: the sql to be executed. (templated) :type sql: string """ @@ -113,7 +113,7 @@ class ValueCheckOperator(BaseOperator): needs to be defined. Whereas a get_db_hook is hook that gets a single record from an external source. - :param sql: the sql to be executed + :param sql: the sql to be executed. (templated) :type sql: string """ http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/operators/docker_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/docker_operator.py b/airflow/operators/docker_operator.py index 8694a3d..9d535ff 100644 --- a/airflow/operators/docker_operator.py +++ b/airflow/operators/docker_operator.py @@ -45,7 +45,7 @@ class DockerOperator(BaseOperator): :param api_version: Remote API version. Set to ``auto`` to automatically detect the server's version. :type api_version: str - :param command: Command to be run in the container. + :param command: Command to be run in the container. (templated) :type command: str or list :param cpus: Number of CPUs to assign to the container. This value gets multiplied with 1024. See @@ -54,7 +54,7 @@ class DockerOperator(BaseOperator): :param docker_url: URL of the host running the docker daemon. Default is unix://var/run/docker.sock :type docker_url: str - :param environment: Environment variables to set in the container. + :param environment: Environment variables to set in the container. (templated) :type environment: dict :param force_pull: Pull the docker image on every run. Default is false. :type force_pull: bool http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/operators/email_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/email_operator.py b/airflow/operators/email_operator.py index 69ed285..4f5396f 100644 --- a/airflow/operators/email_operator.py +++ b/airflow/operators/email_operator.py @@ -26,12 +26,12 @@ class EmailOperator(BaseOperator): """ Sends an email. - :param to: list of emails to send the email to + :param to: list of emails to send the email to. (templated) :type to: list or string (comma or semicolon delimited) - :param subject: subject line for the email (templated) + :param subject: subject line for the email. (templated) :type subject: string - :param html_content: content of the email (templated), html markup - is allowed + :param html_content: content of the email, html markup + is allowed. (templated) :type html_content: string :param files: file names to attach in email :type files: list http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/operators/generic_transfer.py ---------------------------------------------------------------------- diff --git a/airflow/operators/generic_transfer.py b/airflow/operators/generic_transfer.py index 03f30a3..8e0684d 100644 --- a/airflow/operators/generic_transfer.py +++ b/airflow/operators/generic_transfer.py @@ -30,16 +30,16 @@ class GenericTransfer(BaseOperator): This is meant to be used on small-ish datasets that fit in memory. - :param sql: SQL query to execute against the source database + :param sql: SQL query to execute against the source database. (templated) :type sql: str - :param destination_table: target table + :param destination_table: target table. (templated) :type destination_table: str :param source_conn_id: source connection :type source_conn_id: str :param destination_conn_id: source connection :type destination_conn_id: str :param preoperator: sql statement or list of statements to be - executed prior to loading the data + executed prior to loading the data. (templated) :type preoperator: str or list of str """ http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/operators/hive_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/hive_operator.py b/airflow/operators/hive_operator.py index b62744b..c3b248a 100644 --- a/airflow/operators/hive_operator.py +++ b/airflow/operators/hive_operator.py @@ -31,9 +31,10 @@ class HiveOperator(BaseOperator): Executes hql code or hive script in a specific Hive database. :param hql: the hql to be executed. Note that you may also use - a relative path from the dag file of a (template) hive script. + a relative path from the dag file of a (template) hive + script. (templated) :type hql: string - :param hive_cli_conn_id: reference to the Hive database + :param hive_cli_conn_id: reference to the Hive database. (templated) :type hive_cli_conn_id: string :param hiveconfs: if defined, these key value pairs will be passed to hive as ``-hiveconf "key"="value"`` @@ -48,7 +49,7 @@ class HiveOperator(BaseOperator): :param script_begin_tag: If defined, the operator will get rid of the part of the script before the first occurrence of `script_begin_tag` :type script_begin_tag: str - :param mapred_queue: queue used by the Hadoop CapacityScheduler + :param mapred_queue: queue used by the Hadoop CapacityScheduler. (templated) :type mapred_queue: string :param mapred_queue_priority: priority within CapacityScheduler queue. Possible settings include: VERY_HIGH, HIGH, NORMAL, LOW, VERY_LOW http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/operators/hive_stats_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/hive_stats_operator.py b/airflow/operators/hive_stats_operator.py index c0de8b4..721149c 100644 --- a/airflow/operators/hive_stats_operator.py +++ b/airflow/operators/hive_stats_operator.py @@ -42,9 +42,9 @@ class HiveStatsCollectionOperator(BaseOperator): value BIGINT ); - :param table: the source table, in the format ``database.table_name`` + :param table: the source table, in the format ``database.table_name``. (templated) :type table: str - :param partition: the source partition + :param partition: the source partition. (templated) :type partition: dict of {col:value} :param extra_exprs: dict of expression to run against the table where keys are metric names and values are Presto compatible expressions http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/operators/hive_to_druid.py ---------------------------------------------------------------------- diff --git a/airflow/operators/hive_to_druid.py b/airflow/operators/hive_to_druid.py index cec0078..c188ef1 100644 --- a/airflow/operators/hive_to_druid.py +++ b/airflow/operators/hive_to_druid.py @@ -31,7 +31,7 @@ class HiveToDruidTransfer(BaseOperator): into memory before being pushed to Druid, so this operator should be used for smallish amount of data.[/del] - :param sql: SQL query to execute against the Druid database + :param sql: SQL query to execute against the Druid database. (templated) :type sql: str :param druid_datasource: the datasource you want to ingest into in druid :type druid_datasource: str @@ -48,8 +48,8 @@ class HiveToDruidTransfer(BaseOperator): :param hadoop_dependency_coordinates: list of coordinates to squeeze int the ingest json :type hadoop_dependency_coordinates: list of str - :param intervals: list of time intervals that defines segments, this - is passed as is to the json object + :param intervals: list of time intervals that defines segments, + this is passed as is to the json object. (templated) :type intervals: list :param hive_tblproperties: additional properties for tblproperties in hive for the staging table http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/operators/hive_to_mysql.py ---------------------------------------------------------------------- diff --git a/airflow/operators/hive_to_mysql.py b/airflow/operators/hive_to_mysql.py index 164f887..730bfb7 100644 --- a/airflow/operators/hive_to_mysql.py +++ b/airflow/operators/hive_to_mysql.py @@ -30,23 +30,23 @@ class HiveToMySqlTransfer(BaseOperator): into memory before being pushed to MySQL, so this operator should be used for smallish amount of data. - :param sql: SQL query to execute against Hive server + :param sql: SQL query to execute against Hive server. (templated) :type sql: str :param mysql_table: target MySQL table, use dot notation to target a - specific database + specific database. (templated) :type mysql_table: str :param mysql_conn_id: source mysql connection :type mysql_conn_id: str :param hiveserver2_conn_id: destination hive connection :type hiveserver2_conn_id: str :param mysql_preoperator: sql statement to run against mysql prior to - import, typically use to truncate of delete in place of the data - coming in, allowing the task to be idempotent (running the task - twice won't double load data) + import, typically use to truncate of delete in place + of the data coming in, allowing the task to be idempotent (running + the task twice won't double load data). (templated) :type mysql_preoperator: str :param mysql_postoperator: sql statement to run against mysql after the - import, typically used to move data from staging to production - and issue cleanup commands. + import, typically used to move data from staging to + production and issue cleanup commands. (templated) :type mysql_postoperator: str :param bulk_load: flag to use bulk_load option. This loads mysql directly from a tab-delimited text file using the LOAD DATA LOCAL INFILE command. http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/operators/hive_to_samba_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/hive_to_samba_operator.py b/airflow/operators/hive_to_samba_operator.py index ad6ccb3..ef9d810 100644 --- a/airflow/operators/hive_to_samba_operator.py +++ b/airflow/operators/hive_to_samba_operator.py @@ -29,7 +29,7 @@ class Hive2SambaOperator(BaseOperator): Executes hql code in a specific Hive database and loads the results of the query as a csv to a Samba location. - :param hql: the hql to be exported + :param hql: the hql to be exported. (templated) :type hql: string :param hiveserver2_conn_id: reference to the hiveserver2 service :type hiveserver2_conn_id: string http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/operators/http_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/http_operator.py b/airflow/operators/http_operator.py index 258a8a2..0507431 100644 --- a/airflow/operators/http_operator.py +++ b/airflow/operators/http_operator.py @@ -29,12 +29,12 @@ class SimpleHttpOperator(BaseOperator): :param http_conn_id: The connection to run the sensor against :type http_conn_id: string - :param endpoint: The relative part of the full url + :param endpoint: The relative part of the full url. (templated) :type endpoint: string :param method: The HTTP method to use, default = "POST" :type method: string :param data: The data to pass. POST-data in POST/PUT and params - in the URL for a GET request. + in the URL for a GET request. (templated) :type data: For POST/PUT, depends on the content-type parameter, for GET a dictionary of key/value string pairs :param headers: The HTTP headers to be added to the GET request http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/operators/jdbc_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/jdbc_operator.py b/airflow/operators/jdbc_operator.py index e73d2d5..67fd84d 100644 --- a/airflow/operators/jdbc_operator.py +++ b/airflow/operators/jdbc_operator.py @@ -29,7 +29,7 @@ class JdbcOperator(BaseOperator): :param jdbc_conn_id: reference to a predefined database :type jdbc_conn_id: string - :param sql: the sql code to be executed + :param sql: the sql code to be executed. (templated) :type sql: Can receive a str representing a sql statement, a list of str (sql statements), or reference to a template file. Template reference are recognized by str ending in '.sql' http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/operators/mssql_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/mssql_operator.py b/airflow/operators/mssql_operator.py index 0278e6e..9dd7cf6 100644 --- a/airflow/operators/mssql_operator.py +++ b/airflow/operators/mssql_operator.py @@ -28,7 +28,8 @@ class MsSqlOperator(BaseOperator): :param mssql_conn_id: reference to a specific mssql database :type mssql_conn_id: string :param sql: the sql code to be executed - :type sql: string or string pointing to a template file with .sql extension + :type sql: string or string pointing to a template file with .sql + extension. (templated) :param database: name of database which overwrite defined one in connection :type database: string """ http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/dc78b919/airflow/operators/mssql_to_hive.py ---------------------------------------------------------------------- diff --git a/airflow/operators/mssql_to_hive.py b/airflow/operators/mssql_to_hive.py index e5e27ab..e459fd2 100644 --- a/airflow/operators/mssql_to_hive.py +++ b/airflow/operators/mssql_to_hive.py @@ -44,15 +44,18 @@ class MsSqlToHiveTransfer(BaseOperator): stage the data into a temporary table before loading it into its final destination using a ``HiveOperator``. - :param sql: SQL query to execute against the Microsoft SQL Server database + :param sql: SQL query to execute against the Microsoft SQL Server + database. (templated) :type sql: str - :param hive_table: target Hive table, use dot notation to target a specific database + :param hive_table: target Hive table, use dot notation to target a specific + database. (templated) :type hive_table: str :param create: whether to create the table if it doesn't exist :type create: bool :param recreate: whether to drop and recreate the table at every execution :type recreate: bool - :param partition: target partition as a dict of partition columns and values + :param partition: target partition as a dict of partition columns and + values. (templated) :type partition: dict :param delimiter: field delimiter in the file :type delimiter: str
