RussellSpitzer commented on code in PR #7976:
URL: https://github.com/apache/iceberg/pull/7976#discussion_r1290241933
##########
core/src/test/java/org/apache/iceberg/io/TestResolvingIO.java:
##########
@@ -47,4 +63,69 @@ public void testResolvingFileIOJavaSerialization() throws
IOException, ClassNotF
FileIO roundTripSerializedFileIO =
TestHelpers.roundTripSerialize(testResolvingFileIO);
assertThat(roundTripSerializedFileIO.properties()).isEqualTo(testResolvingFileIO.properties());
}
+
+ @Test
+ public void resolveFileIOBulkDeletion() throws IOException {
+ // configure resolving fileIO
+ ResolvingFileIO resolvingFileIO = new ResolvingFileIO();
+ Configuration hadoopConf = new Configuration();
+ resolvingFileIO.setConf(hadoopConf);
+ resolvingFileIO.initialize(
+ ImmutableMap.of("io-impl", "org.apache.iceberg.hadoop.HadoopFileIO"));
+ ResolvingFileIO spy = spy(resolvingFileIO);
+ // configure delegation IO
+ FileSystem fs = FileSystem.getLocal(hadoopConf);
+ Path parent = new Path(tempDir.toURI());
+ HadoopFileIO delegate = new HadoopFileIO(hadoopConf);
+ when(spy.io(anyString())).thenReturn(delegate);
+ // write
+ List<Path> randomFilePaths =
+ IntStream.range(1, 10)
+ .mapToObj(i -> new Path(parent, "random-" + i + "-" +
UUID.randomUUID()))
+ .collect(Collectors.toList());
+ for (Path randomFilePath : randomFilePaths) {
+ fs.createNewFile(randomFilePath);
+
assertThat(delegate.newInputFile(randomFilePath.toUri().toString()).exists()).isTrue();
+ }
+ // bulk deletion
+ List<String> randomFilePathString =
+ randomFilePaths.stream().map(p ->
p.toUri().toString()).collect(Collectors.toList());
+ spy.deleteFiles(randomFilePathString);
+
+ for (String path : randomFilePathString) {
+ assertThat(delegate.newInputFile(path).exists()).isFalse();
+ }
+ }
+
+ @Test
+ public void resolveFileIONonBulkDeletion() {
+ // configure resolving fileIO
+ ResolvingFileIO resolvingFileIO = new ResolvingFileIO();
+ Configuration hadoopConf = new Configuration();
+ resolvingFileIO.setConf(hadoopConf);
+ resolvingFileIO.initialize(
Review Comment:
Do you need to do this? You always return in mmemory file io with .io is
called?
##########
core/src/test/java/org/apache/iceberg/io/TestResolvingIO.java:
##########
@@ -47,4 +63,69 @@ public void testResolvingFileIOJavaSerialization() throws
IOException, ClassNotF
FileIO roundTripSerializedFileIO =
TestHelpers.roundTripSerialize(testResolvingFileIO);
assertThat(roundTripSerializedFileIO.properties()).isEqualTo(testResolvingFileIO.properties());
}
+
+ @Test
+ public void resolveFileIOBulkDeletion() throws IOException {
+ // configure resolving fileIO
+ ResolvingFileIO resolvingFileIO = new ResolvingFileIO();
+ Configuration hadoopConf = new Configuration();
+ resolvingFileIO.setConf(hadoopConf);
+ resolvingFileIO.initialize(
+ ImmutableMap.of("io-impl", "org.apache.iceberg.hadoop.HadoopFileIO"));
+ ResolvingFileIO spy = spy(resolvingFileIO);
+ // configure delegation IO
+ FileSystem fs = FileSystem.getLocal(hadoopConf);
+ Path parent = new Path(tempDir.toURI());
+ HadoopFileIO delegate = new HadoopFileIO(hadoopConf);
+ when(spy.io(anyString())).thenReturn(delegate);
+ // write
+ List<Path> randomFilePaths =
+ IntStream.range(1, 10)
+ .mapToObj(i -> new Path(parent, "random-" + i + "-" +
UUID.randomUUID()))
+ .collect(Collectors.toList());
+ for (Path randomFilePath : randomFilePaths) {
+ fs.createNewFile(randomFilePath);
+
assertThat(delegate.newInputFile(randomFilePath.toUri().toString()).exists()).isTrue();
+ }
+ // bulk deletion
+ List<String> randomFilePathString =
+ randomFilePaths.stream().map(p ->
p.toUri().toString()).collect(Collectors.toList());
+ spy.deleteFiles(randomFilePathString);
+
+ for (String path : randomFilePathString) {
+ assertThat(delegate.newInputFile(path).exists()).isFalse();
+ }
+ }
+
+ @Test
+ public void resolveFileIONonBulkDeletion() {
+ // configure resolving fileIO
+ ResolvingFileIO resolvingFileIO = new ResolvingFileIO();
+ Configuration hadoopConf = new Configuration();
+ resolvingFileIO.setConf(hadoopConf);
+ resolvingFileIO.initialize(
Review Comment:
Do you need to do this? You always return in memory file io with .io is
called?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
