Repository: opennlp Updated Branches: refs/heads/master f8fbfc9fd -> f1f241370
OPENNLP-1018: Add more tests for ObjectStreams.. This closed apache/opennlp#164 Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/f1f24137 Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/f1f24137 Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/f1f24137 Branch: refs/heads/master Commit: f1f241370145a601b6a08709df547428b6a95bdf Parents: f8fbfc9 Author: koji <[email protected]> Authored: Tue Apr 18 13:41:16 2017 +0900 Committer: koji <[email protected]> Committed: Tue Apr 18 13:41:16 2017 +0900 ---------------------------------------------------------------------- .../opennlp/tools/ml/model/FileEventStream.java | 15 +++-- .../tools/ml/model/FileEventStreamTest.java | 69 ++++++++++++++++++++ .../ml/model/RealValueFileEventStreamTest.java | 69 ++++++++++++++++++++ .../opennlp/tools/util/ParagraphStreamTest.java | 26 +++++--- .../tools/util/PlainTextByLineStreamTest.java | 38 ++++++++--- 5 files changed, 192 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/opennlp/blob/f1f24137/opennlp-tools/src/main/java/opennlp/tools/ml/model/FileEventStream.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/FileEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/FileEventStream.java index 7adc36b..6f9e316 100644 --- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/FileEventStream.java +++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/FileEventStream.java @@ -23,6 +23,7 @@ import java.io.FileInputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; +import java.io.Reader; import java.util.StringTokenizer; import opennlp.tools.util.ObjectStream; @@ -33,7 +34,7 @@ import opennlp.tools.util.ObjectStream; */ public class FileEventStream implements ObjectStream<Event> { - protected BufferedReader reader; + protected final BufferedReader reader; /** * Creates a new file event stream from the specified file name. @@ -41,18 +42,18 @@ public class FileEventStream implements ObjectStream<Event> { * @throws IOException When the specified file can not be read. */ public FileEventStream(String fileName, String encoding) throws IOException { - if (encoding == null) { - reader = new BufferedReader(new FileReader(fileName)); - } - else { - reader = new BufferedReader(new InputStreamReader(new FileInputStream(fileName),encoding)); - } + this(encoding == null ? + new FileReader(fileName) : new InputStreamReader(new FileInputStream(fileName), encoding)); } public FileEventStream(String fileName) throws IOException { this(fileName,null); } + public FileEventStream(Reader reader) throws IOException { + this.reader = new BufferedReader(reader); + } + /** * Creates a new file event stream from the specified file. * @param file the file containing the events. http://git-wip-us.apache.org/repos/asf/opennlp/blob/f1f24137/opennlp-tools/src/test/java/opennlp/tools/ml/model/FileEventStreamTest.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/test/java/opennlp/tools/ml/model/FileEventStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/ml/model/FileEventStreamTest.java new file mode 100644 index 0000000..3837a25 --- /dev/null +++ b/opennlp-tools/src/test/java/opennlp/tools/ml/model/FileEventStreamTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.ml.model; + +import java.io.IOException; +import java.io.StringReader; + +import org.junit.Assert; +import org.junit.Test; + +public class FileEventStreamTest { + + private static final String EVENTS = + "other wc=ic w&c=he,ic n1wc=lc n1w&c=belongs,lc n2wc=lc\n" + + "other wc=lc w&c=belongs,lc p1wc=ic p1w&c=he,ic n1wc=lc\n" + + "other wc=lc w&c=to,lc p1wc=lc p1w&c=belongs,lc p2wc=ic\n" + + "org-start wc=ic w&c=apache,ic p1wc=lc p1w&c=to,lc\n" + + "org-cont wc=ic w&c=software,ic p1wc=ic p1w&c=apache,ic\n" + + "org-cont wc=ic w&c=foundation,ic p1wc=ic p1w&c=software,ic\n" + + "other wc=other w&c=.,other p1wc=ic\n"; + + @Test + public void testSimpleReading() throws IOException { + FileEventStream feStream = new FileEventStream(new StringReader(EVENTS)); + + Assert.assertEquals("other [wc=ic w&c=he,ic n1wc=lc n1w&c=belongs,lc n2wc=lc]", + feStream.read().toString()); + Assert.assertEquals("other [wc=lc w&c=belongs,lc p1wc=ic p1w&c=he,ic n1wc=lc]", + feStream.read().toString()); + Assert.assertEquals("other [wc=lc w&c=to,lc p1wc=lc p1w&c=belongs,lc p2wc=ic]", + feStream.read().toString()); + Assert.assertEquals("org-start [wc=ic w&c=apache,ic p1wc=lc p1w&c=to,lc]", + feStream.read().toString()); + Assert.assertEquals("org-cont [wc=ic w&c=software,ic p1wc=ic p1w&c=apache,ic]", + feStream.read().toString()); + Assert.assertEquals("org-cont [wc=ic w&c=foundation,ic p1wc=ic p1w&c=software,ic]", + feStream.read().toString()); + Assert.assertEquals("other [wc=other w&c=.,other p1wc=ic]", + feStream.read().toString()); + Assert.assertNull(feStream.read()); + } + + @Test + public void testReset() throws IOException { + FileEventStream feStream = new FileEventStream(new StringReader(EVENTS)); + + try { + feStream.reset(); + Assert.fail("UnsupportedOperationException should be thrown"); + } + catch (UnsupportedOperationException expected) { + } + } +} http://git-wip-us.apache.org/repos/asf/opennlp/blob/f1f24137/opennlp-tools/src/test/java/opennlp/tools/ml/model/RealValueFileEventStreamTest.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/test/java/opennlp/tools/ml/model/RealValueFileEventStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/ml/model/RealValueFileEventStreamTest.java new file mode 100644 index 0000000..9474dc6 --- /dev/null +++ b/opennlp-tools/src/test/java/opennlp/tools/ml/model/RealValueFileEventStreamTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.ml.model; + +import java.io.IOException; +import java.io.StringReader; + +import org.junit.Assert; +import org.junit.Test; + +public class RealValueFileEventStreamTest { + + private static final String EVENTS = + "other wc=ic=1 w&c=he,ic=2 n1wc=lc=3 n1w&c=belongs,lc=4 n2wc=lc=5\n" + + "other wc=lc=1 w&c=belongs,lc=2 p1wc=ic=3 p1w&c=he,ic=4 n1wc=lc=5\n" + + "other wc=lc=1 w&c=to,lc=2 p1wc=lc=3 p1w&c=belongs,lc=4 p2wc=ic=5\n" + + "org-start wc=ic=1 w&c=apache,ic=2 p1wc=lc=3 p1w&c=to,lc=4\n" + + "org-cont wc=ic=1 w&c=software,ic=2 p1wc=ic=3 p1w&c=apache,ic=4\n" + + "org-cont wc=ic=1 w&c=foundation,ic=2 p1wc=ic=3 p1w&c=software,ic=4\n" + + "other wc=other=1 w&c=.,other=2 p1wc=ic=3\n"; + + @Test + public void testSimpleReading() throws IOException { + FileEventStream feStream = new FileEventStream(new StringReader(EVENTS)); + + Assert.assertEquals("other [wc=ic=1 w&c=he,ic=2 n1wc=lc=3 n1w&c=belongs,lc=4 n2wc=lc=5]", + feStream.read().toString()); + Assert.assertEquals("other [wc=lc=1 w&c=belongs,lc=2 p1wc=ic=3 p1w&c=he,ic=4 n1wc=lc=5]", + feStream.read().toString()); + Assert.assertEquals("other [wc=lc=1 w&c=to,lc=2 p1wc=lc=3 p1w&c=belongs,lc=4 p2wc=ic=5]", + feStream.read().toString()); + Assert.assertEquals("org-start [wc=ic=1 w&c=apache,ic=2 p1wc=lc=3 p1w&c=to,lc=4]", + feStream.read().toString()); + Assert.assertEquals("org-cont [wc=ic=1 w&c=software,ic=2 p1wc=ic=3 p1w&c=apache,ic=4]", + feStream.read().toString()); + Assert.assertEquals("org-cont [wc=ic=1 w&c=foundation,ic=2 p1wc=ic=3 p1w&c=software,ic=4]", + feStream.read().toString()); + Assert.assertEquals("other [wc=other=1 w&c=.,other=2 p1wc=ic=3]", + feStream.read().toString()); + Assert.assertNull(feStream.read()); + } + + @Test + public void testReset() throws IOException { + FileEventStream feStream = new FileEventStream(new StringReader(EVENTS)); + + try { + feStream.reset(); + Assert.fail("UnsupportedOperationException should be thrown"); + } + catch (UnsupportedOperationException expected) { + } + } +} http://git-wip-us.apache.org/repos/asf/opennlp/blob/f1f24137/opennlp-tools/src/test/java/opennlp/tools/util/ParagraphStreamTest.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/ParagraphStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/ParagraphStreamTest.java index 82ba01d..3d0c117 100644 --- a/opennlp-tools/src/test/java/opennlp/tools/util/ParagraphStreamTest.java +++ b/opennlp-tools/src/test/java/opennlp/tools/util/ParagraphStreamTest.java @@ -26,23 +26,31 @@ public class ParagraphStreamTest { @Test public void testSimpleReading() throws IOException { - String line1 = "1"; - String line2 = "2"; - String line3 = ""; - String line4 = "4"; - String line5 = "5"; - String line6 = ""; - ParagraphStream paraStream = new ParagraphStream( - ObjectStreamUtils.createObjectStream(line1, line2, line3, line4, line5)); + ObjectStreamUtils.createObjectStream("1", "2", "", "", "4", "5")); Assert.assertEquals("1\n2\n", paraStream.read()); Assert.assertEquals("4\n5\n", paraStream.read()); + Assert.assertNull(paraStream.read()); paraStream = new ParagraphStream( - ObjectStreamUtils.createObjectStream(line1, line2, line3, line4, line5, line6)); + ObjectStreamUtils.createObjectStream("1", "2", "", "", "4", "5", "")); + + Assert.assertEquals("1\n2\n", paraStream.read()); + Assert.assertEquals("4\n5\n", paraStream.read()); + Assert.assertNull(paraStream.read()); + } + + @Test + public void testReset() throws IOException { + ParagraphStream paraStream = new ParagraphStream( + ObjectStreamUtils.createObjectStream("1", "2", "", "", "4", "5", "")); + + Assert.assertEquals("1\n2\n", paraStream.read()); + paraStream.reset(); Assert.assertEquals("1\n2\n", paraStream.read()); Assert.assertEquals("4\n5\n", paraStream.read()); + Assert.assertNull(paraStream.read()); } } http://git-wip-us.apache.org/repos/asf/opennlp/blob/f1f24137/opennlp-tools/src/test/java/opennlp/tools/util/PlainTextByLineStreamTest.java ---------------------------------------------------------------------- diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/PlainTextByLineStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/PlainTextByLineStreamTest.java index 5307f6f..c551b3a 100644 --- a/opennlp-tools/src/test/java/opennlp/tools/util/PlainTextByLineStreamTest.java +++ b/opennlp-tools/src/test/java/opennlp/tools/util/PlainTextByLineStreamTest.java @@ -28,24 +28,44 @@ import org.junit.Test; */ public class PlainTextByLineStreamTest { + static final String testString = "line1" + + '\n' + + "line2" + + '\n' + + "line3" + + "\r\n" + + "line4" + + '\n'; + @Test public void testLineSegmentation() throws IOException { - String testString = "line1" + - '\n' + - "line2" + - '\n' + - "line3" + - "\r\n" + - "line4" + - '\n'; + ObjectStream<String> stream = + new PlainTextByLineStream(new MockInputStreamFactory(testString), StandardCharsets.UTF_8); + + Assert.assertEquals("line1", stream.read()); + Assert.assertEquals("line2", stream.read()); + Assert.assertEquals("line3", stream.read()); + Assert.assertEquals("line4", stream.read()); + Assert.assertNull(stream.read()); + stream.close(); + } + + @Test + public void testReset() throws IOException { ObjectStream<String> stream = - new PlainTextByLineStream(new MockInputStreamFactory(testString), StandardCharsets.UTF_8); + new PlainTextByLineStream(new MockInputStreamFactory(testString), StandardCharsets.UTF_8); + + Assert.assertEquals("line1", stream.read()); + Assert.assertEquals("line2", stream.read()); + Assert.assertEquals("line3", stream.read()); + stream.reset(); Assert.assertEquals("line1", stream.read()); Assert.assertEquals("line2", stream.read()); Assert.assertEquals("line3", stream.read()); Assert.assertEquals("line4", stream.read()); + Assert.assertNull(stream.read()); stream.close(); }
