This is an automated email from the ASF dual-hosted git repository. ggregory pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/commons-io.git
commit cddc925ea8ec7b37d86df5a067a65035e0892d6e Author: Gary Gregory <gardgreg...@gmail.com> AuthorDate: Mon Jun 20 09:08:08 2022 -0400 Add option for AccumulatorPathVisitor to ignore file visitation failures --- .../commons/io/file/AccumulatorPathVisitor.java | 16 +++ .../commons/io/file/CountingPathVisitor.java | 29 +++++- .../apache/commons/io/file/NoopPathVisitor.java | 24 +++++ .../apache/commons/io/file/SimplePathVisitor.java | 21 ++++ .../io/file/AccumulatorPathVisitorTest.java | 116 +++++++++++++++++++++ 5 files changed, 205 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java b/src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java index c5d2b0de..0574ddd7 100644 --- a/src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java +++ b/src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java @@ -18,6 +18,7 @@ package org.apache.commons.io.file; import java.io.IOException; +import java.nio.file.FileVisitResult; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; @@ -26,6 +27,7 @@ import java.util.List; import java.util.Objects; import org.apache.commons.io.file.Counters.PathCounters; +import org.apache.commons.io.function.IOBiFunction; /** * Accumulates normalized paths during visitation. @@ -135,6 +137,20 @@ public class AccumulatorPathVisitor extends CountingPathVisitor { super(pathCounter, fileFilter, dirFilter); } + /** + * Constructs a new instance. + * + * @param pathCounter How to count path visits. + * @param fileFilter Filters which files to count. + * @param dirFilter Filters which directories to count. + * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}. + * @since 2.12.0 + */ + public AccumulatorPathVisitor(final PathCounters pathCounter, final PathFilter fileFilter, final PathFilter dirFilter, + final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) { + super(pathCounter, fileFilter, dirFilter, visitFileFailed); + } + private void add(final List<Path> list, final Path dir) { list.add(dir.normalize()); } diff --git a/src/main/java/org/apache/commons/io/file/CountingPathVisitor.java b/src/main/java/org/apache/commons/io/file/CountingPathVisitor.java index c0ab0cba..b841019d 100644 --- a/src/main/java/org/apache/commons/io/file/CountingPathVisitor.java +++ b/src/main/java/org/apache/commons/io/file/CountingPathVisitor.java @@ -26,8 +26,10 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.Objects; import org.apache.commons.io.file.Counters.PathCounters; +import org.apache.commons.io.filefilter.IOFileFilter; import org.apache.commons.io.filefilter.SymbolicLinkFileFilter; import org.apache.commons.io.filefilter.TrueFileFilter; +import org.apache.commons.io.function.IOBiFunction; /** * Counts files, directories, and sizes, as a visit proceeds. @@ -38,6 +40,14 @@ public class CountingPathVisitor extends SimplePathVisitor { static final String[] EMPTY_STRING_ARRAY = {}; + static IOFileFilter defaultDirFilter() { + return TrueFileFilter.INSTANCE; + } + + static IOFileFilter defaultFileFilter() { + return new SymbolicLinkFileFilter(FileVisitResult.TERMINATE, FileVisitResult.CONTINUE); + } + /** * Creates a new instance configured with a {@link BigInteger} {@link PathCounters}. * @@ -66,7 +76,7 @@ public class CountingPathVisitor extends SimplePathVisitor { * @param pathCounter How to count path visits. */ public CountingPathVisitor(final PathCounters pathCounter) { - this(pathCounter, new SymbolicLinkFileFilter(FileVisitResult.TERMINATE, FileVisitResult.CONTINUE), TrueFileFilter.INSTANCE); + this(pathCounter, defaultFileFilter(), defaultDirFilter()); } /** @@ -83,6 +93,23 @@ public class CountingPathVisitor extends SimplePathVisitor { this.dirFilter = Objects.requireNonNull(dirFilter, "dirFilter"); } + /** + * Constructs a new instance. + * + * @param pathCounter How to count path visits. + * @param fileFilter Filters which files to count. + * @param dirFilter Filters which directories to count. + * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}. + * @since 2.12.0 + */ + public CountingPathVisitor(final PathCounters pathCounter, final PathFilter fileFilter, final PathFilter dirFilter, + final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) { + super(visitFileFailed); + this.pathCounters = Objects.requireNonNull(pathCounter, "pathCounter"); + this.fileFilter = Objects.requireNonNull(fileFilter, "fileFilter"); + this.dirFilter = Objects.requireNonNull(dirFilter, "dirFilter"); + } + @Override public boolean equals(final Object obj) { if (this == obj) { diff --git a/src/main/java/org/apache/commons/io/file/NoopPathVisitor.java b/src/main/java/org/apache/commons/io/file/NoopPathVisitor.java index 48a13239..8fc07d5f 100644 --- a/src/main/java/org/apache/commons/io/file/NoopPathVisitor.java +++ b/src/main/java/org/apache/commons/io/file/NoopPathVisitor.java @@ -17,6 +17,12 @@ package org.apache.commons.io.file; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Path; + +import org.apache.commons.io.function.IOBiFunction; + /** * A noop path visitor. * @@ -28,4 +34,22 @@ public class NoopPathVisitor extends SimplePathVisitor { * The singleton instance. */ public static final NoopPathVisitor INSTANCE = new NoopPathVisitor(); + + /** + * Constructs a new instance. + * + * @since 2.12.0 + */ + public NoopPathVisitor() { + } + + /** + * Constructs a new instance. + * + * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}. + * @since 2.12.0 + */ + public NoopPathVisitor(final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) { + super(visitFileFailed); + } } diff --git a/src/main/java/org/apache/commons/io/file/SimplePathVisitor.java b/src/main/java/org/apache/commons/io/file/SimplePathVisitor.java index b4bb2d9e..1c3b1ddc 100644 --- a/src/main/java/org/apache/commons/io/file/SimplePathVisitor.java +++ b/src/main/java/org/apache/commons/io/file/SimplePathVisitor.java @@ -17,8 +17,13 @@ package org.apache.commons.io.file; +import java.io.IOException; +import java.nio.file.FileVisitResult; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; +import java.util.Objects; + +import org.apache.commons.io.function.IOBiFunction; /** * A {@link SimpleFileVisitor} typed to a {@link Path}. @@ -27,10 +32,26 @@ import java.nio.file.SimpleFileVisitor; */ public abstract class SimplePathVisitor extends SimpleFileVisitor<Path> implements PathVisitor { + private final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailedFunction; + /** * Constructs a new instance. */ protected SimplePathVisitor() { + this.visitFileFailedFunction = super::visitFileFailed; + } + + /** + * Constructs a new instance. + * + * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}. + */ + protected SimplePathVisitor(final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) { + this.visitFileFailedFunction = Objects.requireNonNull(visitFileFailed, "visitFileFailed"); } + @Override + public FileVisitResult visitFileFailed(final Path file, final IOException exc) throws IOException { + return visitFileFailedFunction.apply(file, exc); + } } diff --git a/src/test/java/org/apache/commons/io/file/AccumulatorPathVisitorTest.java b/src/test/java/org/apache/commons/io/file/AccumulatorPathVisitorTest.java index b03aaaae..06903a1e 100644 --- a/src/test/java/org/apache/commons/io/file/AccumulatorPathVisitorTest.java +++ b/src/test/java/org/apache/commons/io/file/AccumulatorPathVisitorTest.java @@ -22,9 +22,20 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import java.util.stream.Stream; @@ -32,6 +43,7 @@ import org.apache.commons.io.filefilter.AndFileFilter; import org.apache.commons.io.filefilter.DirectoryFileFilter; import org.apache.commons.io.filefilter.EmptyFileFilter; import org.apache.commons.io.filefilter.PathVisitorFileFilter; +import org.apache.commons.io.function.IOBiFunction; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -50,6 +62,17 @@ public class AccumulatorPathVisitorTest { // @formatter:on } + static Stream<Arguments> testParametersIgnoreFailures() { + // @formatter:off + return Stream.of( + Arguments.of((Supplier<AccumulatorPathVisitor>) () -> new AccumulatorPathVisitor( + Counters.bigIntegerPathCounters(), + CountingPathVisitor.defaultDirFilter(), + CountingPathVisitor.defaultFileFilter(), + IOBiFunction.noop()))); + // @formatter:on + } + @TempDir Path tempDirPath; @@ -109,4 +132,97 @@ public class AccumulatorPathVisitorTest { assertEquals(2, accPathVisitor.getFileList().size()); } + /** + * Tests IO-755 with a directory with 100 files, and delete all of them mid-way through the visit. + * + * Random failure like: + * + * <pre> + * ...?... + * </pre> + */ + @ParameterizedTest + @MethodSource("testParametersIgnoreFailures") + public void testFolderWhileDeletingAsync(final Supplier<AccumulatorPathVisitor> supplier) throws IOException, InterruptedException { + final int count = 10_000; + final List<Path> files = new ArrayList<>(count); + // Create "count" file fixtures + for (int i = 1; i <= count; i++) { + final Path tempFile = Files.createTempFile(tempDirPath, "test", ".txt"); + assertTrue(Files.exists(tempFile)); + files.add(tempFile); + } + final AccumulatorPathVisitor accPathVisitor = supplier.get(); + final PathVisitorFileFilter countingFileFilter = new PathVisitorFileFilter(accPathVisitor) { + @Override + public FileVisitResult visitFile(final Path path, final BasicFileAttributes attributes) throws IOException { + // Slow down the walking a bit to try and cause conflicts with the deletion thread + try { + Thread.sleep(10); + } catch (final InterruptedException ignore) { + // e.printStackTrace(); + } + return super.visitFile(path, attributes); + } + }; + final ExecutorService executor = Executors.newSingleThreadExecutor(); + final AtomicBoolean deleted = new AtomicBoolean(); + try { + executor.execute(() -> { + for (final Path file : files) { + try { + // File deletion is slow compared to tree walking, so we go as fast as we can here + Files.delete(file); + } catch (final IOException ignored) { + // e.printStackTrace(); + } + } + deleted.set(true); + }); + Files.walkFileTree(tempDirPath, countingFileFilter); + } finally { + if (!deleted.get()) { + Thread.sleep(1000); + } + if (!deleted.get()) { + executor.awaitTermination(5, TimeUnit.SECONDS); + } + executor.shutdownNow(); + } + } + + /** + * Tests IO-755 with a directory with 100 files, and delete all of them mid-way through the visit. + */ + @ParameterizedTest + @MethodSource("testParametersIgnoreFailures") + public void testFolderWhileDeletingSync(final Supplier<AccumulatorPathVisitor> supplier) throws IOException { + final int count = 100; + final int marker = count / 2; + final Set<Path> files = new LinkedHashSet<>(count); + for (int i = 1; i <= count; i++) { + final Path tempFile = Files.createTempFile(tempDirPath, "test", ".txt"); + assertTrue(Files.exists(tempFile)); + files.add(tempFile); + } + final AccumulatorPathVisitor accPathVisitor = supplier.get(); + final AtomicInteger visitCount = new AtomicInteger(); + final PathVisitorFileFilter countingFileFilter = new PathVisitorFileFilter(accPathVisitor) { + @Override + public FileVisitResult visitFile(final Path path, final BasicFileAttributes attributes) throws IOException { + if (visitCount.incrementAndGet() == marker) { + // Now that we've visited half the files, delete them all + for (final Path file : files) { + Files.delete(file); + } + } + return super.visitFile(path, attributes); + } + }; + Files.walkFileTree(tempDirPath, countingFileFilter); + assertCounts(1, marker - 1, 0, accPathVisitor.getPathCounters()); + assertEquals(1, accPathVisitor.getDirList().size()); + assertEquals(marker - 1, accPathVisitor.getFileList().size()); + } + }