[
https://issues.apache.org/jira/browse/BEAM-5417?focusedWorklogId=147753&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-147753
]
ASF GitHub Bot logged work on BEAM-5417:
----------------------------------------
Author: ASF GitHub Bot
Created on: 25/Sep/18 20:05
Start Date: 25/Sep/18 20:05
Worklog Time Spent: 10m
Work Description: udim commented on a change in pull request #6423:
[BEAM-5417] Parity between GCS and local match
URL: https://github.com/apache/beam/pull/6423#discussion_r220335224
##########
File path: sdks/python/apache_beam/io/filesystem_test.py
##########
@@ -126,66 +180,69 @@ def test_match_glob(self):
('apple/dish/bat', 13),
('apple/dish/cat', 14),
('apple/dish/carl', 15),
+ ('banana/cat', 16),
+ ('banana/cyrano.md', 17),
+ ('banana/cyrano.mb', 18),
]
- for (object_name, size) in objects:
+ bucket_name = 'gcsio-test'
+
+ if expected_object_names is all:
+ # A hack around the fact that the parameterized decorator does not have
+ # access to self.objects
+ expected_object_names = objects
+
+ for object_name, size in objects:
file_name = 'gs://%s/%s' % (bucket_name, object_name)
self.fs._insert_random_file(file_name, size)
- test_cases = [
- ('gs://*', objects),
- ('gs://gcsio-test/*', objects),
- ('gs://gcsio-test/cow/*', [
- ('cow/cat/fish', 2),
- ('cow/cat/blubber', 3),
- ('cow/dog/blubber', 4),
- ]),
- ('gs://gcsio-test/cow/ca*', [
- ('cow/cat/fish', 2),
- ('cow/cat/blubber', 3),
- ]),
- ('gs://gcsio-test/apple/[df]ish/ca*', [
- ('apple/fish/cat', 10),
- ('apple/fish/cart', 11),
- ('apple/fish/carl', 12),
- ('apple/dish/cat', 14),
- ('apple/dish/carl', 15),
- ]),
- ('gs://gcsio-test/apple/fish/car?', [
- ('apple/fish/cart', 11),
- ('apple/fish/carl', 12),
- ]),
- ('gs://gcsio-test/apple/fish/b*', [
- ('apple/fish/blubber', 6),
- ('apple/fish/blowfish', 7),
- ('apple/fish/bambi', 8),
- ('apple/fish/balloon', 9),
- ]),
- ('gs://gcsio-test/apple/f*/b*', [
- ('apple/fish/blubber', 6),
- ('apple/fish/blowfish', 7),
- ('apple/fish/bambi', 8),
- ('apple/fish/balloon', 9),
- ]),
- ('gs://gcsio-test/apple/dish/[cb]at', [
- ('apple/dish/bat', 13),
- ('apple/dish/cat', 14),
- ]),
+
+ expected_file_names = [('gs://%s/%s' % (bucket_name, object_name), size)
+ for object_name, size in expected_object_names]
+ actual_file_names = [
+ (file_metadata.path, file_metadata.size_in_bytes)
+ for file_metadata in self._flatten_match(self.fs.match([file_pattern]))
]
- for file_pattern, expected_object_names in test_cases:
- expected_file_names = [('gs://%s/%s' % (bucket_name, object_name), size)
- for (object_name, size) in expected_object_names]
- self.assertEqual(
- set([(file_metadata.path, file_metadata.size_in_bytes)
- for file_metadata in
- self._flatten_match(self.fs.match([file_pattern]))]),
- set(expected_file_names))
+
+ self.maxDiff = None
+ self.assertEqual(set(actual_file_names), set(expected_file_names))
# Check if limits are followed correctly
limit = 3
- for file_pattern, expected_object_names in test_cases:
- expected_num_items = min(len(expected_object_names), limit)
- self.assertEqual(
- len(self._flatten_match(self.fs.match([file_pattern], [limit]))),
- expected_num_items)
+ expected_num_items = min(len(expected_object_names), limit)
+ self.assertEqual(
+ len(self._flatten_match(self.fs.match([file_pattern], [limit]))),
+ expected_num_items)
+
+ @parameterized.expand([
+ param(os_path=posixpath, sep_re='\\/'),
+ param(os_path=ntpath, sep_re='\\\\'),
+ ])
+ def test_translate_pattern(self, os_path, sep_re):
+ star = r'[^/\\]*'
+ double_star = r'.*'
+ join = os_path.join
+
+ def re_join(a, *p):
+ path = a
+ for b in p:
+ if b.startswith(sep_re):
Review comment:
This branch is not used in any of `pattern__expected`, right?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 147753)
Time Spent: 3h (was: 2h 50m)
> FileSystems.match behaviour diff between GCS and local file system
> ------------------------------------------------------------------
>
> Key: BEAM-5417
> URL: https://issues.apache.org/jira/browse/BEAM-5417
> Project: Beam
> Issue Type: Bug
> Components: sdk-py-core
> Affects Versions: 2.5.0, 2.6.0
> Reporter: Joar Wandborg
> Assignee: Chamikara Jayalath
> Priority: Major
> Time Spent: 3h
> Remaining Estimate: 0h
>
> Given the directory structure:
>
> {noformat}
> .
> ├── filesystem-match-test
> │ ├── a
> │ │ └── file.txt
> │ └── b
> │ └── file.txt
> └── filesystem-match-test.py
> {noformat}
>
> Where {{filesystem-match-test.py}} contains:
> {code:python}
> from __future__ import print_function
> import os
> import posixpath
> from apache_beam.io.filesystem import MatchResult
> from apache_beam.io.filesystems import FileSystems
> BASES = [
> os.path.join(os.path.dirname(__file__), "./"),
> "gs://my-bucket/test/",
> ]
> pattern = "filesystem-match-test/*/file.txt"
> for base_path in BASES:
> full_pattern = posixpath.join(base_path, pattern)
> print("full_pattern: {}".format(full_pattern))
> match_result = FileSystems.match([full_pattern])[0] # type: MatchResult
> print("metadata list: {}".format(match_result.metadata_list))
> {code}
> Running {{python filesystem-match-test.py}} does not match any files locally,
> but does match files on GCS:
> {noformat}
> full_pattern: ./filesystem-match-test/*/file.txt
> metadata list: []
> full_pattern: gs://my-bucket/test/filesystem-match-test/*/file.txt
> metadata list:
> [FileMetadata(gs://my-bucket/test/filesystem-match-test/a/file.txt, 6),
> FileMetadata(gs://my-bucket/test/filesystem-match-test/b/file.txt, 6)]
> {noformat}
> The expected result is that a/file.txt and b/file.txt should be matched for
> both patterns.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
