Repository: incubator-airflow Updated Branches: refs/heads/v1-10-test 5495d2590 -> 16bae5634 (forced update)
[AIRFLOW-2382] Fix wrong description for delimiter Fix misleading descriptions for the 'delimiter' parameter in S3ListOperator and S3ToGoogleCloudStorageOperator's docstring. Closes #3270 from sekikn/AIRFLOW-2382 Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/ae63246a Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/ae63246a Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/ae63246a Branch: refs/heads/v1-10-test Commit: ae63246a1d580d77c366276300139165e4c984de Parents: 36193fc Author: Kengo Seki <[email protected]> Authored: Thu Apr 26 18:45:12 2018 -0700 Committer: r39132 <[email protected]> Committed: Thu Apr 26 18:45:12 2018 -0700 ---------------------------------------------------------------------- airflow/contrib/operators/s3_list_operator.py | 12 +++++------- airflow/contrib/operators/s3_to_gcs_operator.py | 6 ++---- 2 files changed, 7 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/ae63246a/airflow/contrib/operators/s3_list_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/s3_list_operator.py b/airflow/contrib/operators/s3_list_operator.py index 1dcbc60..dbb45fe 100644 --- a/airflow/contrib/operators/s3_list_operator.py +++ b/airflow/contrib/operators/s3_list_operator.py @@ -24,8 +24,7 @@ from airflow.utils.decorators import apply_defaults class S3ListOperator(BaseOperator): """ - List all objects from the bucket with the given string prefix and delimiter - in name. + List all objects from the bucket with the given string prefix in name. This operator returns a python list with the name of objects which can be used by `xcom` in the downstream task. @@ -35,22 +34,21 @@ class S3ListOperator(BaseOperator): :param prefix: Prefix string to filters the objects whose name begin with such prefix :type prefix: string - :param delimiter: The delimiter by which you want to filter the objects. - For e.g to lists the CSV files from in a directory in S3 you would use - delimiter='.csv'. + :param delimiter: the delimiter marks key hierarchy. :type delimiter: string :param aws_conn_id: The connection ID to use when connecting to S3 storage. :type aws_conn_id: string **Example**: - The following operator would list all the CSV files from the S3 + The following operator would list all the files + (excluding subfolders) from the S3 ``customers/2018/04/`` key in the ``data`` bucket. :: s3_file = S3ListOperator( task_id='list_3s_files', bucket='data', prefix='customers/2018/04/', - delimiter='.csv', + delimiter='/', aws_conn_id='aws_customers_conn' ) """ http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/ae63246a/airflow/contrib/operators/s3_to_gcs_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/s3_to_gcs_operator.py b/airflow/contrib/operators/s3_to_gcs_operator.py index d105596..5a2004d 100644 --- a/airflow/contrib/operators/s3_to_gcs_operator.py +++ b/airflow/contrib/operators/s3_to_gcs_operator.py @@ -37,12 +37,10 @@ class S3ToGoogleCloudStorageOperator(S3ListOperator): :param prefix: Prefix string which filters objects whose name begin with such prefix. :type prefix: string - :param delimiter: The delimiter by which you want to filter the objects on. - E.g. to list CSV files from a S3 key you would do the following, - `delimiter='.csv'`. + :param delimiter: the delimiter marks key hierarchy. :type delimiter: string :param aws_conn_id: The source S3 connection - :type aws_conn_id: str + :type aws_conn_id: string :param dest_gcs_conn_id: The destination connection ID to use when connecting to Google Cloud Storage. :type dest_gcs_conn_id: string
