Repository: opennlp Updated Branches: refs/heads/master cff6e0009 -> 27214015c
OPENNLP-1024:Add unit tests and javadocs for DirectorySampleStream, this closes apache/opennlp#160 Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/27214015 Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/27214015 Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/27214015 Branch: refs/heads/master Commit: 27214015c725619e0098e227fd29816d0891d4c5 Parents: cff6e00 Author: jzonthemtn <[email protected]> Authored: Sun Apr 16 17:24:16 2017 -0400 Committer: smarthi <[email protected]> Committed: Sun Apr 16 17:24:19 2017 -0400 ---------------------------------------------------------------------- .../tools/formats/DirectorySampleStream.java | 34 +++- .../formats/DirectorySampleStreamTest.java | 187 +++++++++++++++++++ 2 files changed, 215 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/opennlp/blob/27214015/opennlp-tools/src/main/java/opennlp/tools/formats/DirectorySampleStream.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/DirectorySampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/DirectorySampleStream.java index 3a5621a..da73507 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/formats/DirectorySampleStream.java +++ b/opennlp-tools/src/main/java/opennlp/tools/formats/DirectorySampleStream.java @@ -28,24 +28,30 @@ import java.util.Stack; import opennlp.tools.util.ObjectStream; /** - * The directory sample stream scans a directory (recursively) for plain text - * files and outputs each file as a String object. + * The directory sample stream allows for creating a stream + * from a directory listing of files. */ public class DirectorySampleStream implements ObjectStream<File> { private final List<File> inputDirectories; - private final boolean isRecursiveScan; + private final boolean recursive; private final FileFilter fileFilter; private Stack<File> directories = new Stack<>(); private Stack<File> textFiles = new Stack<>(); - + + /** + * Creates a new directory sample stream. + * @param dirs The directories to read. + * @param fileFilter The {@link FileFilter filter} to apply while enumerating files. + * @param recursive Enables or disables recursive file listing. + */ public DirectorySampleStream(File[] dirs, FileFilter fileFilter, boolean recursive) { this.fileFilter = fileFilter; - isRecursiveScan = recursive; + this.recursive = recursive; List<File> inputDirectoryList = new ArrayList<>(dirs.length); @@ -64,10 +70,17 @@ public class DirectorySampleStream implements ObjectStream<File> { directories.addAll(inputDirectories); } + /** + * Creates a new directory sample stream. + * @param dir The {@link File directory}. + * @param fileFilter The {@link FileFilter filter} to apply while enumerating files. + * @param recursive Enables or disables recursive file listing. + */ public DirectorySampleStream(File dir, FileFilter fileFilter, boolean recursive) { this(new File[]{dir}, fileFilter, recursive); } + @Override public File read() throws IOException { while (textFiles.isEmpty() && !directories.isEmpty()) { @@ -86,7 +99,7 @@ public class DirectorySampleStream implements ObjectStream<File> { if (file.isFile()) { textFiles.push(file); } - else if (isRecursiveScan && file.isDirectory()) { + else if (recursive && file.isDirectory()) { directories.push(file); } } @@ -100,6 +113,7 @@ public class DirectorySampleStream implements ObjectStream<File> { } } + @Override public void reset() { directories.clear(); textFiles.clear(); @@ -107,6 +121,14 @@ public class DirectorySampleStream implements ObjectStream<File> { directories.addAll(inputDirectories); } + /** + * {@inheritDoc} + * Calling this function has no effect on + * the stream. + */ + @Override public void close() throws IOException { + } + } http://git-wip-us.apache.org/repos/asf/opennlp/blob/27214015/opennlp-tools/src/test/java/opennlp/tools/formats/DirectorySampleStreamTest.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/test/java/opennlp/tools/formats/DirectorySampleStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/formats/DirectorySampleStreamTest.java new file mode 100644 index 0000000..d17188e --- /dev/null +++ b/opennlp-tools/src/test/java/opennlp/tools/formats/DirectorySampleStreamTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.formats; + +import java.io.File; +import java.io.FileFilter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class DirectorySampleStreamTest { + + @Rule + public TemporaryFolder tempDirectory = new TemporaryFolder(); + + @Test + public void directoryTest() throws IOException { + + FileFilter filter = new TempFileNameFilter(); + + List<File> files = new ArrayList<>(); + + File temp1 = tempDirectory.newFile(); + files.add(temp1); + + File temp2 = tempDirectory.newFile(); + files.add(temp2); + + DirectorySampleStream stream = new DirectorySampleStream(tempDirectory.getRoot(), filter, false); + + File file = stream.read(); + Assert.assertTrue(files.contains(file)); + + file = stream.read(); + Assert.assertTrue(files.contains(file)); + + file = stream.read(); + Assert.assertNull(file); + + stream.close(); + + } + + @Test + public void directoryNullFilterTest() throws IOException { + + List<File> files = new ArrayList<>(); + + File temp1 = tempDirectory.newFile(); + files.add(temp1); + + File temp2 = tempDirectory.newFile(); + files.add(temp2); + + DirectorySampleStream stream = new DirectorySampleStream(tempDirectory.getRoot(), null, false); + + File file = stream.read(); + Assert.assertTrue(files.contains(file)); + + file = stream.read(); + Assert.assertTrue(files.contains(file)); + + file = stream.read(); + Assert.assertNull(file); + + stream.close(); + + } + + @Test + public void recursiveDirectoryTest() throws IOException { + + FileFilter filter = new TempFileNameFilter(); + + List<File> files = new ArrayList<>(); + + File temp1 = tempDirectory.newFile(); + files.add(temp1); + + File tempSubDirectory = tempDirectory.newFolder("sub1"); + File temp2 = File.createTempFile("sub1", ".tmp", tempSubDirectory); + files.add(temp2); + + DirectorySampleStream stream = new DirectorySampleStream(tempDirectory.getRoot(), filter, true); + + File file = stream.read(); + Assert.assertTrue(files.contains(file)); + + file = stream.read(); + Assert.assertTrue(files.contains(file)); + + file = stream.read(); + Assert.assertNull(file); + + stream.close(); + + } + + @Test + public void resetDirectoryTest() throws IOException { + + FileFilter filter = new TempFileNameFilter(); + + List<File> files = new ArrayList<>(); + + File temp1 = tempDirectory.newFile(); + files.add(temp1); + + File temp2 = tempDirectory.newFile(); + files.add(temp2); + + DirectorySampleStream stream = new DirectorySampleStream(tempDirectory.getRoot(), filter, false); + + File file = stream.read(); + Assert.assertTrue(files.contains(file)); + + stream.reset(); + + file = stream.read(); + Assert.assertTrue(files.contains(file)); + + file = stream.read(); + Assert.assertTrue(files.contains(file)); + + file = stream.read(); + Assert.assertNull(file); + + stream.close(); + + } + + @Test + public void emptyDirectoryTest() throws IOException { + + FileFilter filter = new TempFileNameFilter(); + + DirectorySampleStream stream = new DirectorySampleStream(tempDirectory.getRoot(), filter, false); + + Assert.assertNull(stream.read()); + + stream.close(); + + } + + @Test(expected = IllegalArgumentException.class) + public void invalidDirectoryTest() throws IOException { + + FileFilter filter = new TempFileNameFilter(); + + DirectorySampleStream stream = new DirectorySampleStream(tempDirectory.newFile(), filter, false); + + Assert.assertNull(stream.read()); + + stream.close(); + + } + + class TempFileNameFilter implements FileFilter { + + @Override + public boolean accept(File file) { + return file.isDirectory() || file.getName().endsWith(".tmp"); + } + + } + +}
