Repository: incubator-airflow Updated Branches: refs/heads/master 8e83e2b3e -> b5f758bb6
[AIRFLOW-2328] Fix empty GCS blob in S3ToGoogleCloudStorageOperator Closes #3231 from wileeam/fix-empty-blob-in-s3-to- gcs-operator Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/b5f758bb Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/b5f758bb Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/b5f758bb Branch: refs/heads/master Commit: b5f758bb618af2878cf90e5418d3abc8c234b45f Parents: 8e83e2b Author: Guillermo RodrÃguez Cano <[email protected]> Authored: Sat Apr 21 08:36:38 2018 +0200 Committer: Fokko Driesprong <[email protected]> Committed: Sat Apr 21 08:36:38 2018 +0200 ---------------------------------------------------------------------- airflow/contrib/operators/s3_to_gcs_operator.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/b5f758bb/airflow/contrib/operators/s3_to_gcs_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/s3_to_gcs_operator.py b/airflow/contrib/operators/s3_to_gcs_operator.py index d48e468..d105596 100644 --- a/airflow/contrib/operators/s3_to_gcs_operator.py +++ b/airflow/contrib/operators/s3_to_gcs_operator.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -102,11 +102,12 @@ class S3ToGoogleCloudStorageOperator(S3ListOperator): self.replace = replace if dest_gcs and not self._gcs_object_is_directory(self.dest_gcs): - self.log.info('Destination Google Cloud Storage path is not a ' - 'valid "directory", define one and end the path ' - 'with a slash: "/".') + self.log.info( + 'Destination Google Cloud Storage path is not a valid ' + '"directory", define a path that ends with a slash "/" or ' + 'leave it empty for the root of the bucket.') raise AirflowException('The destination Google Cloud Storage path ' - 'must end with a slash "/".') + 'must end with a slash "/" or be empty.') def execute(self, context): # use the super method to list all the files in an S3 bucket/key @@ -188,4 +189,4 @@ class S3ToGoogleCloudStorageOperator(S3ListOperator): def _gcs_object_is_directory(self, object): bucket, blob = _parse_gcs_url(object) - return blob.endswith('/') + return len(blob) == 0 or blob.endswith('/')
