kou opened a new issue, #37555: URL: https://github.com/apache/arrow/issues/37555
### Describe the bug, including details regarding any error messages, version, and platform. https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/builds/47954752 ```text _ test_get_file_info_with_selector[PyFileSystem(FSSpecHandler(fsspec.LocalFileSystem()))] _ fs = <pyarrow._fs.PyFileSystem object at 0x00000230E8CD5720> pathfn = <function py_fsspec_localfs.<locals>.<lambda> at 0x00000230E4A47490> def test_get_file_info_with_selector(fs, pathfn): base_dir = pathfn('selector-dir/') file_a = pathfn('selector-dir/test_file_a') file_b = pathfn('selector-dir/test_file_b') dir_a = pathfn('selector-dir/test_dir_a') file_c = pathfn('selector-dir/test_dir_a/test_file_c') dir_b = pathfn('selector-dir/test_dir_b') try: fs.create_dir(base_dir) with fs.open_output_stream(file_a): pass with fs.open_output_stream(file_b): pass fs.create_dir(dir_a) with fs.open_output_stream(file_c): pass fs.create_dir(dir_b) # recursive selector selector = FileSelector(base_dir, allow_not_found=False, recursive=True) assert selector.base_dir == base_dir infos = fs.get_file_info(selector) if fs.type_name == "py::fsspec+s3": # s3fs only lists directories if they are not empty, but depending # on the s3fs/fsspec version combo, it includes the base_dir # (https://github.com/dask/s3fs/issues/393) assert (len(infos) == 4) or (len(infos) == 5) else: > assert len(infos) == 5 E AssertionError: assert 6 == 5 E + where 6 = len([<FileInfo for 'C:/Users/appveyor/AppData/Local/Temp/1/pytest-of-appveyor/pytest-0/test_get_file_info_with_select8/sel...p/1/pytest-of-appveyor/pytest-0/test_get_file_info_with_select8/selector-dir/test_file_b': type=FileType.File, size=0>]) pyarrow\tests\test_fs.py:707: AssertionError _ test_get_file_info_with_selector[PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))] _ fs = <pyarrow._fs.PyFileSystem object at 0x00000230E5148100> pathfn = <function py_fsspec_memoryfs.<locals>.<lambda> at 0x00000230E4A47750> def test_get_file_info_with_selector(fs, pathfn): base_dir = pathfn('selector-dir/') file_a = pathfn('selector-dir/test_file_a') file_b = pathfn('selector-dir/test_file_b') dir_a = pathfn('selector-dir/test_dir_a') file_c = pathfn('selector-dir/test_dir_a/test_file_c') dir_b = pathfn('selector-dir/test_dir_b') try: fs.create_dir(base_dir) with fs.open_output_stream(file_a): pass with fs.open_output_stream(file_b): pass fs.create_dir(dir_a) with fs.open_output_stream(file_c): pass fs.create_dir(dir_b) # recursive selector selector = FileSelector(base_dir, allow_not_found=False, recursive=True) assert selector.base_dir == base_dir infos = fs.get_file_info(selector) if fs.type_name == "py::fsspec+s3": # s3fs only lists directories if they are not empty, but depending # on the s3fs/fsspec version combo, it includes the base_dir # (https://github.com/dask/s3fs/issues/393) assert (len(infos) == 4) or (len(infos) == 5) else: > assert len(infos) == 5 E AssertionError: assert 6 == 5 E + where 6 = len([<FileInfo for '/selector-dir': type=FileType.Directory>, <FileInfo for '/selector-dir/test_dir_a': type=FileType.Dire...-dir/test_file_a': type=FileType.File, size=0>, <FileInfo for '/selector-dir/test_file_b': type=FileType.File, size=0>]) pyarrow\tests\test_fs.py:707: AssertionError _ test_get_file_info_with_selector[PyFileSystem(FSSpecHandler(s3fs.S3FileSystem()))] _ fs = <pyarrow._fs.PyFileSystem object at 0x00000230E4971EA0> pathfn = <method-wrapper '__add__' of str object at 0x00000230DA38F9A0> def test_get_file_info_with_selector(fs, pathfn): base_dir = pathfn('selector-dir/') file_a = pathfn('selector-dir/test_file_a') file_b = pathfn('selector-dir/test_file_b') dir_a = pathfn('selector-dir/test_dir_a') file_c = pathfn('selector-dir/test_dir_a/test_file_c') dir_b = pathfn('selector-dir/test_dir_b') try: fs.create_dir(base_dir) with fs.open_output_stream(file_a): pass with fs.open_output_stream(file_b): pass fs.create_dir(dir_a) with fs.open_output_stream(file_c): pass fs.create_dir(dir_b) # recursive selector selector = FileSelector(base_dir, allow_not_found=False, recursive=True) assert selector.base_dir == base_dir infos = fs.get_file_info(selector) if fs.type_name == "py::fsspec+s3": # s3fs only lists directories if they are not empty, but depending # on the s3fs/fsspec version combo, it includes the base_dir # (https://github.com/dask/s3fs/issues/393) assert (len(infos) == 4) or (len(infos) == 5) else: assert len(infos) == 5 for info in infos: if (info.path.endswith(file_a) or info.path.endswith(file_b) or info.path.endswith(file_c)): assert info.type == FileType.File elif (info.path.rstrip("/").endswith(dir_a) or info.path.rstrip("/").endswith(dir_b)): assert info.type == FileType.Directory elif (fs.type_name == "py::fsspec+s3" and info.path.rstrip("/").endswith("selector-dir")): # s3fs can include base dir, see above assert info.type == FileType.Directory else: raise ValueError('unexpected path {}'.format(info.path)) check_mtime_or_absent(info) # non-recursive selector -> not selecting the nested file_c selector = FileSelector(base_dir, recursive=False) infos = fs.get_file_info(selector) if fs.type_name == "py::fsspec+s3": # s3fs only lists directories if they are not empty # + for s3fs 0.5.2 all directories are dropped because of buggy # side-effect of previous find() call # (https://github.com/dask/s3fs/issues/410) > assert (len(infos) == 3) or (len(infos) == 2) E AssertionError: assert (4 == 3 or 4 == 2) E + where 4 = len([<FileInfo for 'pyarrow-filesystem/selector-dir': type=FileType.Directory>, <FileInfo for 'pyarrow-filesystem/selector... type=FileType.File, size=0>, <FileInfo for 'pyarrow-filesystem/selector-dir/test_file_b': type=FileType.File, size=0>]) E + and 4 = len([<FileInfo for 'pyarrow-filesystem/selector-dir': type=FileType.Directory>, <FileInfo for 'pyarrow-filesystem/selector... type=FileType.File, size=0>, <FileInfo for 'pyarrow-filesystem/selector-dir/test_file_b': type=FileType.File, size=0>]) pyarrow\tests\test_fs.py:733: AssertionError ``` Note that the last succeeded build uses "fsspec 2023.6.0": https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/builds/47946265?fullLog=true#L460 And the failed build uses "fsspec 2023.9.0": https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/builds/47954752?fullLog=true#L461 ### Component(s) Continuous Integration, Python -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
