raulcd opened a new issue, #37675:
URL: https://github.com/apache/arrow/issues/37675

   ### Describe the bug, including details regarding any error messages, 
version, and platform.
   
   As seen here on main:
   
https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/builds/48012043
   and here on a PR:
   
https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/builds/48015180
   
   Currently our Appveyor CI is failing on 
`python/pyarrow/tests/test_fs.py::test_get_file_info_with_selector` with:
   ```
   ================================== FAILURES 
===================================
   _ 
test_get_file_info_with_selector[PyFileSystem(FSSpecHandler(fsspec.LocalFileSystem()))]
 _
   fs = <pyarrow._fs.PyFileSystem object at 0x0000027D2A466CE0>
   pathfn = <function py_fsspec_localfs.<locals>.<lambda> at 0x0000027D2A373E30>
       def test_get_file_info_with_selector(fs, pathfn):
           base_dir = pathfn('selector-dir/')
           file_a = pathfn('selector-dir/test_file_a')
           file_b = pathfn('selector-dir/test_file_b')
           dir_a = pathfn('selector-dir/test_dir_a')
           file_c = pathfn('selector-dir/test_dir_a/test_file_c')
           dir_b = pathfn('selector-dir/test_dir_b')
       
           try:
               fs.create_dir(base_dir)
               with fs.open_output_stream(file_a):
                   pass
               with fs.open_output_stream(file_b):
                   pass
               fs.create_dir(dir_a)
               with fs.open_output_stream(file_c):
                   pass
               fs.create_dir(dir_b)
       
               # recursive selector
               selector = FileSelector(base_dir, allow_not_found=False,
                                       recursive=True)
               assert selector.base_dir == base_dir
       
               infos = fs.get_file_info(selector)
               if fs.type_name == "py::fsspec+s3":
                   # s3fs only lists directories if they are not empty, but 
depending
                   # on the s3fs/fsspec version combo, it includes the base_dir
                   # (https://github.com/dask/s3fs/issues/393)
                   assert (len(infos) == 4) or (len(infos) == 5)
               else:
   >               assert len(infos) == 5
   E               AssertionError: assert 6 == 5
   E                +  where 6 = len([<FileInfo for 
'C:/Users/appveyor/AppData/Local/Temp/1/pytest-of-appveyor/pytest-0/test_get_file_info_with_select8/sel...p/1/pytest-of-appveyor/pytest-0/test_get_file_info_with_select8/selector-dir/test_file_b':
 type=FileType.File, size=0>])
   pyarrow\tests\test_fs.py:707: AssertionError
   _ 
test_get_file_info_with_selector[PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))]
 _
   fs = <pyarrow._fs.PyFileSystem object at 0x0000027D2A681000>
   pathfn = <function py_fsspec_memoryfs.<locals>.<lambda> at 
0x0000027D2A7381B0>
       def test_get_file_info_with_selector(fs, pathfn):
           base_dir = pathfn('selector-dir/')
           file_a = pathfn('selector-dir/test_file_a')
           file_b = pathfn('selector-dir/test_file_b')
           dir_a = pathfn('selector-dir/test_dir_a')
           file_c = pathfn('selector-dir/test_dir_a/test_file_c')
           dir_b = pathfn('selector-dir/test_dir_b')
       
           try:
               fs.create_dir(base_dir)
               with fs.open_output_stream(file_a):
                   pass
               with fs.open_output_stream(file_b):
                   pass
               fs.create_dir(dir_a)
               with fs.open_output_stream(file_c):
                   pass
               fs.create_dir(dir_b)
       
               # recursive selector
               selector = FileSelector(base_dir, allow_not_found=False,
                                       recursive=True)
               assert selector.base_dir == base_dir
       
               infos = fs.get_file_info(selector)
               if fs.type_name == "py::fsspec+s3":
                   # s3fs only lists directories if they are not empty, but 
depending
                   # on the s3fs/fsspec version combo, it includes the base_dir
                   # (https://github.com/dask/s3fs/issues/393)
                   assert (len(infos) == 4) or (len(infos) == 5)
               else:
   >               assert len(infos) == 5
   E               AssertionError: assert 6 == 5
   E                +  where 6 = len([<FileInfo for '/selector-dir': 
type=FileType.Directory>, <FileInfo for '/selector-dir/test_dir_a': 
type=FileType.Dire...-dir/test_file_a': type=FileType.File, size=0>, <FileInfo 
for '/selector-dir/test_file_b': type=FileType.File, size=0>])
   pyarrow\tests\test_fs.py:707: AssertionError
   _ 
test_get_file_info_with_selector[PyFileSystem(FSSpecHandler(s3fs.S3FileSystem()))]
 _
   fs = <pyarrow._fs.PyFileSystem object at 0x0000027D2A687E80>
   pathfn = <method-wrapper '__add__' of str object at 0x0000027D1CFF0760>
       def test_get_file_info_with_selector(fs, pathfn):
           base_dir = pathfn('selector-dir/')
           file_a = pathfn('selector-dir/test_file_a')
           file_b = pathfn('selector-dir/test_file_b')
           dir_a = pathfn('selector-dir/test_dir_a')
           file_c = pathfn('selector-dir/test_dir_a/test_file_c')
           dir_b = pathfn('selector-dir/test_dir_b')
       
           try:
               fs.create_dir(base_dir)
               with fs.open_output_stream(file_a):
                   pass
               with fs.open_output_stream(file_b):
                   pass
               fs.create_dir(dir_a)
               with fs.open_output_stream(file_c):
                   pass
               fs.create_dir(dir_b)
       
               # recursive selector
               selector = FileSelector(base_dir, allow_not_found=False,
                                       recursive=True)
               assert selector.base_dir == base_dir
       
               infos = fs.get_file_info(selector)
               if fs.type_name == "py::fsspec+s3":
                   # s3fs only lists directories if they are not empty, but 
depending
                   # on the s3fs/fsspec version combo, it includes the base_dir
                   # (https://github.com/dask/s3fs/issues/393)
                   assert (len(infos) == 4) or (len(infos) == 5)
               else:
                   assert len(infos) == 5
       
               for info in infos:
                   if (info.path.endswith(file_a) or info.path.endswith(file_b) 
or
                           info.path.endswith(file_c)):
                       assert info.type == FileType.File
                   elif (info.path.rstrip("/").endswith(dir_a) or
                         info.path.rstrip("/").endswith(dir_b)):
                       assert info.type == FileType.Directory
                   elif (fs.type_name == "py::fsspec+s3" and
                         info.path.rstrip("/").endswith("selector-dir")):
                       # s3fs can include base dir, see above
                       assert info.type == FileType.Directory
                   else:
                       raise ValueError('unexpected path {}'.format(info.path))
                   check_mtime_or_absent(info)
       
               # non-recursive selector -> not selecting the nested file_c
               selector = FileSelector(base_dir, recursive=False)
       
               infos = fs.get_file_info(selector)
               if fs.type_name == "py::fsspec+s3":
                   # s3fs only lists directories if they are not empty
                   # + for s3fs 0.5.2 all directories are dropped because of 
buggy
                   # side-effect of previous find() call
                   # (https://github.com/dask/s3fs/issues/410)
   >               assert (len(infos) == 3) or (len(infos) == 2)
   E               AssertionError: assert (4 == 3 or 4 == 2)
   E                +  where 4 = len([<FileInfo for 
'pyarrow-filesystem/selector-dir': type=FileType.Directory>, <FileInfo for 
'pyarrow-filesystem/selector... type=FileType.File, size=0>, <FileInfo for 
'pyarrow-filesystem/selector-dir/test_file_b': type=FileType.File, size=0>])
   E                +  and   4 = len([<FileInfo for 
'pyarrow-filesystem/selector-dir': type=FileType.Directory>, <FileInfo for 
'pyarrow-filesystem/selector... type=FileType.File, size=0>, <FileInfo for 
'pyarrow-filesystem/selector-dir/test_file_b': type=FileType.File, size=0
   ```
   
   ### Component(s)
   
   Continuous Integration, Python


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to