rdblue commented on code in PR #5011:
URL: https://github.com/apache/iceberg/pull/5011#discussion_r901912220
##########
python/tests/conftest.py:
##########
@@ -301,3 +319,81 @@ def all_avro_types() -> Dict[str, Any]:
@pytest.fixture
def catalog() -> InMemoryCatalog:
return InMemoryCatalog("test.in.memory.catalog", {"test.key":
"test.value"})
+
+
[email protected](scope="session")
+def simple_struct():
+ return StructType(
+ NestedField(1, "required_field", StringType(), True, "this is a doc"),
NestedField(2, "optional_field", IntegerType())
+ )
+
+
[email protected](scope="session")
+def simple_list():
+ return ListType(element_id=22, element=StringType(), element_required=True)
+
+
[email protected](scope="session")
+def simple_map():
+ return MapType(key_id=19, key_type=StringType(), value_id=25,
value_type=DoubleType(), value_required=False)
+
+
+class LocalOutputFile(OutputFile):
+ """An OutputFile implementation for local files (for test use only)"""
+
+ def __init__(self, location: str):
+
+ parsed_location = urlparse(location) # Create a ParseResult from the
uri
+ if parsed_location.scheme and parsed_location.scheme != "file": #
Validate that a uri is provided with a scheme of `file`
+ raise ValueError("LocalOutputFile location must have a scheme of
`file`")
+ elif parsed_location.netloc:
+ raise ValueError(f"Network location is not allowed for
LocalOutputFile: {parsed_location.netloc}")
+
+ super().__init__(location=location)
+ self._parsed_location = parsed_location
+
+ @property
+ def parsed_location(self) -> ParseResult:
+ """The parsed location
+ Returns:
+ ParseResult: The parsed results which has attributes `scheme`,
`netloc`, `path`,
+ `params`, `query`, and `fragments`.
+ """
+ return self._parsed_location
+
+ def __len__(self):
+ return os.path.getsize(self.parsed_location.path)
+
+ def exists(self):
+ return os.path.exists(self.parsed_location.path)
+
+ def to_input_file(self):
+ return LocalInputFile(location=self.location)
+
+ def create(self, overwrite: bool = False) -> OutputStream:
+ output_file = open(self.parsed_location.path, "wb" if overwrite else
"xb")
+ if not isinstance(output_file, OutputStream):
+ raise TypeError("Object returned from LocalOutputFile.create(...)
does not match the OutputStream protocol.")
+ return output_file
+
+
+class LocalFileIO(FileIO):
+ """A FileIO implementation for local files (for test use only)"""
+
+ def new_input(self, location: str):
+ return LocalInputFile(location=location)
+
+ def new_output(self, location: str):
+ return LocalOutputFile(location=location)
+
+ def delete(self, location: Union[str, InputFile, OutputFile]):
+ location = location.location if isinstance(location, (InputFile,
OutputFile)) else location
+ try:
+ os.remove(location)
+ except FileNotFoundError as e:
+ raise FileNotFoundError(f"Cannot delete file, does not exist:
{location}") from e
Review Comment:
If the file doesn't exist, why not just return? It doesn't seem worth
throwing an exception.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
