eeroel commented on code in PR #37868:
URL: https://github.com/apache/arrow/pull/37868#discussion_r1404008102
##########
python/pyarrow/_dataset.pyx:
##########
@@ -96,27 +96,33 @@ def _get_parquet_symbol(name):
return _dataset_pq and getattr(_dataset_pq, name)
-cdef CFileSource _make_file_source(object file, FileSystem filesystem=None):
+cdef CFileSource _make_file_source(object file, FileSystem filesystem=None,
int64_t file_size=-1):
cdef:
CFileSource c_source
shared_ptr[CFileSystem] c_filesystem
+ CFileInfo c_info
c_string c_path
shared_ptr[CRandomAccessFile] c_file
shared_ptr[CBuffer] c_buffer
if isinstance(file, Buffer):
c_buffer = pyarrow_unwrap_buffer(file)
c_source = CFileSource(move(c_buffer))
-
elif _is_path_like(file):
if filesystem is None:
raise ValueError("cannot construct a FileSource from "
"a path without a FileSystem")
c_filesystem = filesystem.unwrap()
c_path = tobytes(_stringify_path(file))
- c_source = CFileSource(move(c_path), move(c_filesystem))
+ if file_size >= 0:
+ c_size = file_size
+ info = FileInfo(c_path, size=c_size)
+ c_info = info.unwrap()
Review Comment:
Hmm tried this but got an error that CFileInfo expects one argument. Left it
as is for now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
