[
https://issues.apache.org/jira/browse/BEAM-5486?focusedWorklogId=147764&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-147764
]
ASF GitHub Bot logged work on BEAM-5486:
----------------------------------------
Author: ASF GitHub Bot
Created on: 25/Sep/18 20:25
Start Date: 25/Sep/18 20:25
Worklog Time Spent: 10m
Work Description: pabloem closed pull request #6480: [BEAM-5486] GCSIO:
Allow empty object prefix in list_prefix().
URL: https://github.com/apache/beam/pull/6480
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py
b/sdks/python/apache_beam/io/gcp/gcsio.py
index 7e8a9f01f76..bd6d96f3dd3 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio.py
@@ -142,10 +142,10 @@ def get_new_http():
timeout=DEFAULT_HTTP_TIMEOUT_SECONDS)
-def parse_gcs_path(gcs_path):
+def parse_gcs_path(gcs_path, object_optional=False):
"""Return the bucket and object names of the given gs:// path."""
- match = re.match('^gs://([^/]+)/(.+)$', gcs_path)
- if match is None:
+ match = re.match('^gs://([^/]+)/(.*)$', gcs_path)
+ if match is None or (match.group(2) == '' and not object_optional):
raise ValueError('GCS path must be in the form gs://<bucket>/<object>.')
return match.group(1), match.group(2)
@@ -433,12 +433,12 @@ def list_prefix(self, path):
"""Lists files matching the prefix.
Args:
- path: GCS file path pattern in the form gs://<bucket>/<name>.
+ path: GCS file path pattern in the form gs://<bucket>/[name].
Returns:
Dictionary of file name -> size.
"""
- bucket, prefix = parse_gcs_path(path)
+ bucket, prefix = parse_gcs_path(path, object_optional=True)
request = storage.StorageObjectsListRequest(bucket=bucket, prefix=prefix)
file_sizes = {}
counter = 0
diff --git a/sdks/python/apache_beam/io/gcp/gcsio_test.py
b/sdks/python/apache_beam/io/gcp/gcsio_test.py
index 1e217a6381f..e0af7c962b5 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio_test.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio_test.py
@@ -219,6 +219,14 @@ def Execute(self, unused_http, **unused_kwargs): #
pylint: disable=invalid-name
@unittest.skipIf(HttpError is None, 'GCP dependencies are not installed')
class TestGCSPathParser(unittest.TestCase):
+ BAD_GCS_PATHS = [
+ 'gs://',
+ 'gs://bucket',
+ 'gs:///name',
+ 'gs:///',
+ 'gs:/blah/bucket/name',
+ ]
+
def test_gcs_path(self):
self.assertEqual(
gcsio.parse_gcs_path('gs://bucket/name'), ('bucket', 'name'))
@@ -226,12 +234,21 @@ def test_gcs_path(self):
gcsio.parse_gcs_path('gs://bucket/name/sub'), ('bucket', 'name/sub'))
def test_bad_gcs_path(self):
- self.assertRaises(ValueError, gcsio.parse_gcs_path, 'gs://')
- self.assertRaises(ValueError, gcsio.parse_gcs_path, 'gs://bucket')
+ for path in self.BAD_GCS_PATHS:
+ self.assertRaises(ValueError, gcsio.parse_gcs_path, path)
self.assertRaises(ValueError, gcsio.parse_gcs_path, 'gs://bucket/')
- self.assertRaises(ValueError, gcsio.parse_gcs_path, 'gs:///name')
- self.assertRaises(ValueError, gcsio.parse_gcs_path, 'gs:///')
- self.assertRaises(ValueError, gcsio.parse_gcs_path, 'gs:/blah/bucket/name')
+
+ def test_gcs_path_object_optional(self):
+ self.assertEqual(
+ gcsio.parse_gcs_path('gs://bucket/name', object_optional=True),
+ ('bucket', 'name'))
+ self.assertEqual(
+ gcsio.parse_gcs_path('gs://bucket/', object_optional=True),
+ ('bucket', ''))
+
+ def test_bad_gcs_path_object_optional(self):
+ for path in self.BAD_GCS_PATHS:
+ self.assertRaises(ValueError, gcsio.parse_gcs_path, path, True)
@unittest.skipIf(HttpError is None, 'GCP dependencies are not installed')
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 147764)
Time Spent: 50m (was: 40m)
> Python: Filesystems.match(['gs://bucket/*']) fails
> --------------------------------------------------
>
> Key: BEAM-5486
> URL: https://issues.apache.org/jira/browse/BEAM-5486
> Project: Beam
> Issue Type: Bug
> Components: sdk-py-core
> Reporter: Udi Meiri
> Assignee: Udi Meiri
> Priority: Major
> Time Spent: 50m
> Remaining Estimate: 0h
>
> Reported here: https://github.com/apache/beam/pull/5024#issuecomment-406211816
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
