rdblue commented on code in PR #6392:
URL: https://github.com/apache/iceberg/pull/6392#discussion_r1044923034
##########
python/tests/io/test_fsspec.py:
##########
@@ -204,6 +204,191 @@ def test_writing_avro_file(generated_manifest_entry_file:
Generator[str, None, N
b2 = in_f.read()
assert b1 == b2 # Check that bytes of read from local avro file
match bytes written to s3
+ fsspec_fileio.delete(f"s3://warehouse/{filename}")
+
+
[email protected]
+def test_fsspec_new_input_file_adlfs(adlfs_fsspec_fileio: FsspecFileIO) ->
None:
+ """Test creating a new input file from an fsspec file-io"""
+ filename = str(uuid.uuid4())
+
+ input_file = adlfs_fsspec_fileio.new_input(f"abfss://tests/{filename}")
+
+ assert isinstance(input_file, fsspec.FsspecInputFile)
+ assert input_file.location == f"abfss://tests/{filename}"
+
+
[email protected]
+def test_fsspec_new_abfss_output_file_adlfs(adlfs_fsspec_fileio: FsspecFileIO)
-> None:
+ """Test creating a new output file from an fsspec file-io"""
+ filename = str(uuid.uuid4())
+
+ output_file = adlfs_fsspec_fileio.new_output(f"abfss://tests/{filename}")
+
+ assert isinstance(output_file, fsspec.FsspecOutputFile)
+ assert output_file.location == f"abfss://tests/{filename}"
+
+
[email protected]
+def test_fsspec_write_and_read_file_adlfs(adlfs_fsspec_fileio: FsspecFileIO)
-> None:
+ """Test writing and reading a file using FsspecInputFile and
FsspecOutputFile"""
+ filename = str(uuid.uuid4())
+ output_file =
adlfs_fsspec_fileio.new_output(location=f"abfss://tests/{filename}")
+ with output_file.create() as f:
+ f.write(b"foo")
+
+ input_file = adlfs_fsspec_fileio.new_input(f"abfss://tests/{filename}")
+ assert input_file.open().read() == b"foo"
+
+ adlfs_fsspec_fileio.delete(input_file)
+
+
[email protected]
+def test_fsspec_getting_length_of_file_adlfs(adlfs_fsspec_fileio:
FsspecFileIO) -> None:
+ """Test getting the length of an FsspecInputFile and FsspecOutputFile"""
+ filename = str(uuid.uuid4())
+
+ output_file =
adlfs_fsspec_fileio.new_output(location=f"abfss://tests/{filename}")
+ with output_file.create() as f:
+ f.write(b"foobar")
+
+ assert len(output_file) == 6
+
+ input_file =
adlfs_fsspec_fileio.new_input(location=f"abfss://tests/{filename}")
+ assert len(input_file) == 6
+
+ adlfs_fsspec_fileio.delete(output_file)
+
+
[email protected]
+def test_fsspec_file_tell_adlfs(adlfs_fsspec_fileio: FsspecFileIO) -> None:
+ """Test finding cursor position for an fsspec file-io file"""
+
+ filename = str(uuid.uuid4())
+
+ output_file =
adlfs_fsspec_fileio.new_output(location=f"abfss://tests/{filename}")
+ with output_file.create() as write_file:
+ write_file.write(b"foobar")
+
+ input_file =
adlfs_fsspec_fileio.new_input(location=f"abfss://tests/{filename}")
+ f = input_file.open()
+
+ f.seek(0)
+ assert f.tell() == 0
+ f.seek(1)
+ assert f.tell() == 1
+ f.seek(3)
+ assert f.tell() == 3
+ f.seek(0)
+ assert f.tell() == 0
+
+ adlfs_fsspec_fileio.delete(f"abfss://tests/{filename}")
+
+
[email protected]
+def test_fsspec_read_specified_bytes_for_file_adlfs(adlfs_fsspec_fileio:
FsspecFileIO) -> None:
+ """Test reading a specified number of bytes from an fsspec file-io file"""
+
+ filename = str(uuid.uuid4())
+ output_file =
adlfs_fsspec_fileio.new_output(location=f"abfss://tests/{filename}")
+ with output_file.create() as write_file:
+ write_file.write(b"foo")
+
+ input_file =
adlfs_fsspec_fileio.new_input(location=f"abfss://tests/{filename}")
+ f = input_file.open()
+
+ f.seek(0)
+ assert b"f" == f.read(1)
+ f.seek(0)
+ assert b"fo" == f.read(2)
+ f.seek(1)
+ assert b"o" == f.read(1)
+ f.seek(1)
+ assert b"oo" == f.read(2)
+ f.seek(0)
+ assert b"foo" == f.read(999) # test reading amount larger than entire
content length
+
+ adlfs_fsspec_fileio.delete(input_file)
+
+
[email protected]
+def test_fsspec_raise_on_opening_file_not_found_adlfs(adlfs_fsspec_fileio:
FsspecFileIO) -> None:
+ """Test that an fsspec input file raises appropriately when the adlfs file
is not found"""
+
+ filename = str(uuid.uuid4())
+ input_file =
adlfs_fsspec_fileio.new_input(location=f"abfss://tests/{filename}")
+ with pytest.raises(FileNotFoundError):
+ input_file.open().read()
+
+ # filename is not propagated in FileNotFoundError for adlfs
Review Comment:
Can we add the filename in the `open` or `read` call by catching
`FileNotFoundError` and wrapping it? That seems worth an check in exception
handling:
```python
def open(self):
try:
# original method contents
except FileNotFoundError as e:
if str(e) contains self.location:
raise e
else:
raise FileNotFoundError(f"No such file: {self.location}") from e
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
