Updated Branches: refs/heads/master a97e8e101 -> 687894564
CRUNCH-221: Ignore hidden files during materialization. Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/68789456 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/68789456 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/68789456 Branch: refs/heads/master Commit: 6878945645bd15036bd15c1e4b5a17ab6691fc11 Parents: a97e8e1 Author: Josh Wills <[email protected]> Authored: Wed Jun 19 13:05:15 2013 -0700 Committer: Josh Wills <[email protected]> Committed: Wed Jun 19 13:05:15 2013 -0700 ---------------------------------------------------------------------- .../crunch/io/CompositePathIterableIT.java | 19 +++++++++++++++++++ .../apache/crunch/io/CompositePathIterable.java | 3 ++- 2 files changed, 21 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/68789456/crunch-core/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java b/crunch-core/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java index 08d226d..22f5a8e 100644 --- a/crunch-core/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/io/CompositePathIterableIT.java @@ -81,4 +81,23 @@ public class CompositePathIterableIT { Writables.strings())); } + @Test + public void testCreate_HiddenFiles() throws IOException { + File file = tmpDir.copyResourceFile("set1.txt"); + assertTrue(file.renameTo(new File(tmpDir.getRootFile(), "_set1.txt"))); + + file = tmpDir.copyResourceFile("set1.txt"); + assertTrue(file.renameTo(new File(tmpDir.getRootFile(), ".set1.txt"))); + + Path emptyInputDir = tmpDir.getRootPath(); + + Configuration conf = new Configuration(); + LocalFileSystem local = FileSystem.getLocal(conf); + + Iterable<String> iterable = CompositePathIterable.create(local, emptyInputDir, + new TextFileReaderFactory<String>(Writables.strings())); + + assertTrue(Lists.newArrayList(iterable).isEmpty()); + } + } http://git-wip-us.apache.org/repos/asf/crunch/blob/68789456/crunch-core/src/main/java/org/apache/crunch/io/CompositePathIterable.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/CompositePathIterable.java b/crunch-core/src/main/java/org/apache/crunch/io/CompositePathIterable.java index a4723e9..c36a506 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/CompositePathIterable.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/CompositePathIterable.java @@ -38,7 +38,8 @@ public class CompositePathIterable<T> implements Iterable<T> { private static final PathFilter FILTER = new PathFilter() { @Override public boolean accept(Path path) { - return !path.getName().startsWith("_"); + String name = path.getName(); + return !name.startsWith("_") && !name.startsWith("."); } };
