Repository: incubator-airflow Updated Branches: refs/heads/master d200f6008 -> 5ac8ff105
[AIRFLOW-406] Sphinx/rst fixes Dear Airflow Maintainers, - Fix some syntax errors in our sphinx/rst docstrings, which appears on pythonhosted.org - Move `xcom_push` documentation from constructor to class docstring - Rewrite some copy in `HivePartitionSensor` and `NamedHivePartitionSensor` docstrings - Fix `AirflowImporter` docstring that seems to have been automatically search-and-replaced Closes #1717 from zodiac/xuanji/fix_documentation Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/5ac8ff10 Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/5ac8ff10 Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/5ac8ff10 Branch: refs/heads/master Commit: 5ac8ff105e070d4aaecffd9eaec1e042826326e8 Parents: d200f60 Author: Li Xuanji <xua...@gmail.com> Authored: Wed Aug 10 17:25:59 2016 -0700 Committer: Dan Davydov <dan.davy...@airbnb.com> Committed: Wed Aug 10 17:26:03 2016 -0700 ---------------------------------------------------------------------- airflow/contrib/operators/bigquery_check_operator.py | 5 +++-- airflow/hooks/druid_hook.py | 5 +++-- airflow/operators/bash_operator.py | 8 ++++---- airflow/operators/check_operator.py | 5 +++-- airflow/operators/presto_check_operator.py | 5 +++-- airflow/operators/sensors.py | 15 ++++++++------- airflow/utils/helpers.py | 8 +++++--- 7 files changed, 29 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/5ac8ff10/airflow/contrib/operators/bigquery_check_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/bigquery_check_operator.py b/airflow/contrib/operators/bigquery_check_operator.py index 10f0b7c..1ff8031 100644 --- a/airflow/contrib/operators/bigquery_check_operator.py +++ b/airflow/contrib/operators/bigquery_check_operator.py @@ -25,8 +25,9 @@ class BigQueryCheckOperator(CheckOperator): values return ``False`` the check is failed and errors out. Note that Python bool casting evals the following as ``False``: - * False - * 0 + + * ``False`` + * ``0`` * Empty string (``""``) * Empty list (``[]``) * Empty dictionary or set (``{}``) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/5ac8ff10/airflow/hooks/druid_hook.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/druid_hook.py b/airflow/hooks/druid_hook.py index 5fc5eec..61d3c40 100644 --- a/airflow/hooks/druid_hook.py +++ b/airflow/hooks/druid_hook.py @@ -165,8 +165,9 @@ class DruidHook(BaseHook): intervals, num_shards, target_partition_size, metric_spec=None, hadoop_dependency_coordinates=None): """ load data to druid from hdfs - :params ts_dim: The column name to use as a timestamp - :params metric_spec: A list of dictionaries + + :param ts_dim: The column name to use as a timestamp + :param metric_spec: A list of dictionaries """ task_id = self.send_ingest_query( datasource, static_path, ts_dim, columns, metric_spec, http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/5ac8ff10/airflow/operators/bash_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/bash_operator.py b/airflow/operators/bash_operator.py index a3b1b92..8e30da4 100644 --- a/airflow/operators/bash_operator.py +++ b/airflow/operators/bash_operator.py @@ -31,6 +31,9 @@ class BashOperator(BaseOperator): :param bash_command: The command, set of commands or reference to a bash script (must be '.sh') to be executed. :type bash_command: string + :param xcom_push: If xcom_push is True, the last line written to stdout + will also be pushed to an XCom when the bash command completes. + :type xcom_push: bool :param env: If env is not None, it must be a mapping that defines the environment variables for the new process; these are used instead of inheriting the current process environment, which is the default @@ -50,10 +53,7 @@ class BashOperator(BaseOperator): env=None, output_encoding='utf-8', *args, **kwargs): - """ - If xcom_push is True, the last line written to stdout will also - be pushed to an XCom when the bash command completes. - """ + super(BashOperator, self).__init__(*args, **kwargs) self.bash_command = bash_command self.env = env http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/5ac8ff10/airflow/operators/check_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/check_operator.py b/airflow/operators/check_operator.py index 83190eb..139e451 100644 --- a/airflow/operators/check_operator.py +++ b/airflow/operators/check_operator.py @@ -30,8 +30,9 @@ class CheckOperator(BaseOperator): values return ``False`` the check is failed and errors out. Note that Python bool casting evals the following as ``False``: - * False - * 0 + + * ``False`` + * ``0`` * Empty string (``""``) * Empty list (``[]``) * Empty dictionary or set (``{}``) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/5ac8ff10/airflow/operators/presto_check_operator.py ---------------------------------------------------------------------- diff --git a/airflow/operators/presto_check_operator.py b/airflow/operators/presto_check_operator.py index 6207460..6dfcdec 100644 --- a/airflow/operators/presto_check_operator.py +++ b/airflow/operators/presto_check_operator.py @@ -25,8 +25,9 @@ class PrestoCheckOperator(CheckOperator): values return ``False`` the check is failed and errors out. Note that Python bool casting evals the following as ``False``: - * False - * 0 + + * ``False`` + * ``0`` * Empty string (``""``) * Empty list (``[]``) * Empty dictionary or set (``{}``) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/5ac8ff10/airflow/operators/sensors.py ---------------------------------------------------------------------- diff --git a/airflow/operators/sensors.py b/airflow/operators/sensors.py index e40292a..467c468 100644 --- a/airflow/operators/sensors.py +++ b/airflow/operators/sensors.py @@ -245,10 +245,11 @@ class NamedHivePartitionSensor(BaseSensorOperator): :param partition_names: List of fully qualified names of the partitions to wait for. A fully qualified name is of the - form schema.table/pk1=pv1/pk2=pv2, for example, + form ``schema.table/pk1=pv1/pk2=pv2``, for example, default.users/ds=2016-01-01. This is passed as is to the metastore - Thrift client "get_partitions_by_name" method. Note that - you cannot use logical operators as in HivePartitionSensor. + Thrift client ``get_partitions_by_name`` method. Note that + you cannot use logical or comparison operators as in + HivePartitionSensor. :type partition_names: list of strings :param metastore_conn_id: reference to the metastore thrift service connection id @@ -312,7 +313,7 @@ class HivePartitionSensor(BaseSensorOperator): """ Waits for a partition to show up in Hive. - Note: Because @partition supports general logical operators, it + Note: Because ``partition`` supports general logical operators, it can be inefficient. Consider using NamedHivePartitionSensor instead if you don't need the full flexibility of HivePartitionSensor. @@ -320,9 +321,9 @@ class HivePartitionSensor(BaseSensorOperator): notation (my_database.my_table) :type table: string :param partition: The partition clause to wait for. This is passed as - is to the metastore Thrift client "get_partitions_by_filter" method, - and apparently supports SQL like notation as in `ds='2015-01-01' - AND type='value'` and > < sings as in "ds>=2015-01-01" + is to the metastore Thrift client ``get_partitions_by_filter`` method, + and apparently supports SQL like notation as in ``ds='2015-01-01' + AND type='value'`` and comparison operators as in ``"ds>=2015-01-01"`` :type partition: string :param metastore_conn_id: reference to the metastore thrift service connection id http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/5ac8ff10/airflow/utils/helpers.py ---------------------------------------------------------------------- diff --git a/airflow/utils/helpers.py b/airflow/utils/helpers.py index 517d5f1..bd26f20 100644 --- a/airflow/utils/helpers.py +++ b/airflow/utils/helpers.py @@ -177,13 +177,15 @@ def pprinttable(rows): class AirflowImporter(object): """ Importer that dynamically loads a class and module from its parent. This - allows Airflow to support `from airflow.operators.bash_operator import - BashOperator` even though BashOperator is actually in - airflow.operators.bash_operator. + allows Airflow to support ``from airflow.operators import BashOperator`` + even though BashOperator is actually in + ``airflow.operators.bash_operator``. The importer also takes over for the parent_module by wrapping it. This is required to support attribute-based usage: + .. code:: python + from airflow import operators operators.BashOperator(...) """