kou opened a new issue, #37555:
URL: https://github.com/apache/arrow/issues/37555

   ### Describe the bug, including details regarding any error messages, 
version, and platform.
   
   
https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/builds/47954752
   
   ```text
   _ 
test_get_file_info_with_selector[PyFileSystem(FSSpecHandler(fsspec.LocalFileSystem()))]
 _
   fs = <pyarrow._fs.PyFileSystem object at 0x00000230E8CD5720>
   pathfn = <function py_fsspec_localfs.<locals>.<lambda> at 0x00000230E4A47490>
       def test_get_file_info_with_selector(fs, pathfn):
           base_dir = pathfn('selector-dir/')
           file_a = pathfn('selector-dir/test_file_a')
           file_b = pathfn('selector-dir/test_file_b')
           dir_a = pathfn('selector-dir/test_dir_a')
           file_c = pathfn('selector-dir/test_dir_a/test_file_c')
           dir_b = pathfn('selector-dir/test_dir_b')
       
           try:
               fs.create_dir(base_dir)
               with fs.open_output_stream(file_a):
                   pass
               with fs.open_output_stream(file_b):
                   pass
               fs.create_dir(dir_a)
               with fs.open_output_stream(file_c):
                   pass
               fs.create_dir(dir_b)
       
               # recursive selector
               selector = FileSelector(base_dir, allow_not_found=False,
                                       recursive=True)
               assert selector.base_dir == base_dir
       
               infos = fs.get_file_info(selector)
               if fs.type_name == "py::fsspec+s3":
                   # s3fs only lists directories if they are not empty, but 
depending
                   # on the s3fs/fsspec version combo, it includes the base_dir
                   # (https://github.com/dask/s3fs/issues/393)
                   assert (len(infos) == 4) or (len(infos) == 5)
               else:
   >               assert len(infos) == 5
   E               AssertionError: assert 6 == 5
   E                +  where 6 = len([<FileInfo for 
'C:/Users/appveyor/AppData/Local/Temp/1/pytest-of-appveyor/pytest-0/test_get_file_info_with_select8/sel...p/1/pytest-of-appveyor/pytest-0/test_get_file_info_with_select8/selector-dir/test_file_b':
 type=FileType.File, size=0>])
   pyarrow\tests\test_fs.py:707: AssertionError
   _ 
test_get_file_info_with_selector[PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))]
 _
   fs = <pyarrow._fs.PyFileSystem object at 0x00000230E5148100>
   pathfn = <function py_fsspec_memoryfs.<locals>.<lambda> at 
0x00000230E4A47750>
       def test_get_file_info_with_selector(fs, pathfn):
           base_dir = pathfn('selector-dir/')
           file_a = pathfn('selector-dir/test_file_a')
           file_b = pathfn('selector-dir/test_file_b')
           dir_a = pathfn('selector-dir/test_dir_a')
           file_c = pathfn('selector-dir/test_dir_a/test_file_c')
           dir_b = pathfn('selector-dir/test_dir_b')
       
           try:
               fs.create_dir(base_dir)
               with fs.open_output_stream(file_a):
                   pass
               with fs.open_output_stream(file_b):
                   pass
               fs.create_dir(dir_a)
               with fs.open_output_stream(file_c):
                   pass
               fs.create_dir(dir_b)
       
               # recursive selector
               selector = FileSelector(base_dir, allow_not_found=False,
                                       recursive=True)
               assert selector.base_dir == base_dir
       
               infos = fs.get_file_info(selector)
               if fs.type_name == "py::fsspec+s3":
                   # s3fs only lists directories if they are not empty, but 
depending
                   # on the s3fs/fsspec version combo, it includes the base_dir
                   # (https://github.com/dask/s3fs/issues/393)
                   assert (len(infos) == 4) or (len(infos) == 5)
               else:
   >               assert len(infos) == 5
   E               AssertionError: assert 6 == 5
   E                +  where 6 = len([<FileInfo for '/selector-dir': 
type=FileType.Directory>, <FileInfo for '/selector-dir/test_dir_a': 
type=FileType.Dire...-dir/test_file_a': type=FileType.File, size=0>, <FileInfo 
for '/selector-dir/test_file_b': type=FileType.File, size=0>])
   pyarrow\tests\test_fs.py:707: AssertionError
   _ 
test_get_file_info_with_selector[PyFileSystem(FSSpecHandler(s3fs.S3FileSystem()))]
 _
   fs = <pyarrow._fs.PyFileSystem object at 0x00000230E4971EA0>
   pathfn = <method-wrapper '__add__' of str object at 0x00000230DA38F9A0>
       def test_get_file_info_with_selector(fs, pathfn):
           base_dir = pathfn('selector-dir/')
           file_a = pathfn('selector-dir/test_file_a')
           file_b = pathfn('selector-dir/test_file_b')
           dir_a = pathfn('selector-dir/test_dir_a')
           file_c = pathfn('selector-dir/test_dir_a/test_file_c')
           dir_b = pathfn('selector-dir/test_dir_b')
       
           try:
               fs.create_dir(base_dir)
               with fs.open_output_stream(file_a):
                   pass
               with fs.open_output_stream(file_b):
                   pass
               fs.create_dir(dir_a)
               with fs.open_output_stream(file_c):
                   pass
               fs.create_dir(dir_b)
       
               # recursive selector
               selector = FileSelector(base_dir, allow_not_found=False,
                                       recursive=True)
               assert selector.base_dir == base_dir
       
               infos = fs.get_file_info(selector)
               if fs.type_name == "py::fsspec+s3":
                   # s3fs only lists directories if they are not empty, but 
depending
                   # on the s3fs/fsspec version combo, it includes the base_dir
                   # (https://github.com/dask/s3fs/issues/393)
                   assert (len(infos) == 4) or (len(infos) == 5)
               else:
                   assert len(infos) == 5
       
               for info in infos:
                   if (info.path.endswith(file_a) or info.path.endswith(file_b) 
or
                           info.path.endswith(file_c)):
                       assert info.type == FileType.File
                   elif (info.path.rstrip("/").endswith(dir_a) or
                         info.path.rstrip("/").endswith(dir_b)):
                       assert info.type == FileType.Directory
                   elif (fs.type_name == "py::fsspec+s3" and
                         info.path.rstrip("/").endswith("selector-dir")):
                       # s3fs can include base dir, see above
                       assert info.type == FileType.Directory
                   else:
                       raise ValueError('unexpected path {}'.format(info.path))
                   check_mtime_or_absent(info)
       
               # non-recursive selector -> not selecting the nested file_c
               selector = FileSelector(base_dir, recursive=False)
       
               infos = fs.get_file_info(selector)
               if fs.type_name == "py::fsspec+s3":
                   # s3fs only lists directories if they are not empty
                   # + for s3fs 0.5.2 all directories are dropped because of 
buggy
                   # side-effect of previous find() call
                   # (https://github.com/dask/s3fs/issues/410)
   >               assert (len(infos) == 3) or (len(infos) == 2)
   E               AssertionError: assert (4 == 3 or 4 == 2)
   E                +  where 4 = len([<FileInfo for 
'pyarrow-filesystem/selector-dir': type=FileType.Directory>, <FileInfo for 
'pyarrow-filesystem/selector... type=FileType.File, size=0>, <FileInfo for 
'pyarrow-filesystem/selector-dir/test_file_b': type=FileType.File, size=0>])
   E                +  and   4 = len([<FileInfo for 
'pyarrow-filesystem/selector-dir': type=FileType.Directory>, <FileInfo for 
'pyarrow-filesystem/selector... type=FileType.File, size=0>, <FileInfo for 
'pyarrow-filesystem/selector-dir/test_file_b': type=FileType.File, size=0>])
   pyarrow\tests\test_fs.py:733: AssertionError
   ```
   
   Note that the last succeeded build uses "fsspec 2023.6.0": 
https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/builds/47946265?fullLog=true#L460
   And the failed build uses "fsspec 2023.9.0": 
https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/builds/47954752?fullLog=true#L461
   
   ### Component(s)
   
   Continuous Integration, Python


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to