specify UTF-8 encoding in Scanner line break support
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/f7c7508f Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/f7c7508f Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/f7c7508f Branch: refs/heads/master Commit: f7c7508f0b6aad327794e22771b208a5ec3dbca4 Parents: 06fd10e Author: sblackmon <[email protected]> Authored: Fri Dec 5 12:32:35 2014 -0600 Committer: sblackmon <[email protected]> Committed: Fri Dec 5 12:32:35 2014 -0600 ---------------------------------------------------------------------- .../streams-provider-datasift/pom.xml | 30 ++++++++------------ .../serializer/DatasiftEventClassifierTest.java | 14 +++++++-- 2 files changed, 24 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/f7c7508f/streams-contrib/streams-provider-datasift/pom.xml ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/pom.xml b/streams-contrib/streams-provider-datasift/pom.xml index 0753325..fe7257d 100644 --- a/streams-contrib/streams-provider-datasift/pom.xml +++ b/streams-contrib/streams-provider-datasift/pom.xml @@ -123,6 +123,18 @@ </sources> </configuration> </execution> + <execution> + <id>add-test-source</id> + <phase>generate-sources</phase> + <goals> + <goal>add-test-source</goal> + </goals> + <configuration> + <sources> + <source>src/test/java17</source> + </sources> + </configuration> + </execution> </executions> </plugin> <plugin> @@ -145,24 +157,6 @@ </execution> </executions> </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <executions> - <execution> - <id>add-test-source</id> - <phase>generate-sources</phase> - <goals> - <goal>add-test-source</goal> - </goals> - <configuration> - <sources> - <source>src/test/java17</source> - </sources> - </configuration> - </execution> - </executions> - </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/f7c7508f/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftEventClassifierTest.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftEventClassifierTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftEventClassifierTest.java index fda57c4..6a880b8 100644 --- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftEventClassifierTest.java +++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftEventClassifierTest.java @@ -27,7 +27,9 @@ import org.apache.streams.datasift.util.StreamsDatasiftMapper; import org.apache.streams.jackson.StreamsJacksonMapper; import org.junit.Test; +import java.io.InputStream; import java.util.Scanner; +import java.util.regex.Pattern; /** * Created by sblackmon on 12/13/13. @@ -36,9 +38,14 @@ public class DatasiftEventClassifierTest { private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance(Lists.newArrayList(StreamsDatasiftMapper.DATASIFT_FORMAT)); + protected static Pattern newLinePattern = Pattern.compile("(\\r\\n?|\\n)", Pattern.MULTILINE); + @Test public void testTwitterDetection() throws Exception { - Scanner scanner = new Scanner(DatasiftActivitySerializerTest.class.getResourceAsStream("/twitter_datasift_json.txt")); + + InputStream testFileStream = DatasiftTwitterActivitySerializerTest.class.getResourceAsStream("/twitter_datasift_json.txt"); + Scanner scanner = new Scanner(testFileStream, "UTF-8").useDelimiter(newLinePattern); + String line = null; while(scanner.hasNextLine()) { line = scanner.nextLine(); @@ -50,7 +57,10 @@ public class DatasiftEventClassifierTest { @Test public void testInstagramDetection() throws Exception { - Scanner scanner = new Scanner(DatasiftActivitySerializerTest.class.getResourceAsStream("/instagram_datasift_json.txt")); + + InputStream testFileStream = DatasiftTwitterActivitySerializerTest.class.getResourceAsStream("/instagram_datasift_json.txt"); + Scanner scanner = new Scanner(testFileStream, "UTF-8").useDelimiter(newLinePattern); + String line = null; while(scanner.hasNextLine()) { line = scanner.nextLine();
