specify UTF-8 encoding in Scanner line break support
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/06fd10ef Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/06fd10ef Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/06fd10ef Branch: refs/heads/master Commit: 06fd10ef796c5d308f9a2c2bbc2561a5316c8749 Parents: b57903f Author: sblackmon <sblack...@w2odigital.com> Authored: Thu Dec 4 15:56:29 2014 -0600 Committer: sblackmon <sblack...@w2odigital.com> Committed: Thu Dec 4 15:56:29 2014 -0600 ---------------------------------------------------------------------- .../streams-provider-datasift/pom.xml | 48 ++++++++------------ .../DatasiftActivitySerializerTest.java | 9 +++- ...DatasiftInstagramActivitySerializerTest.java | 6 ++- ...tasiftInteractionActivitySerializerTest.java | 6 ++- .../DatasiftTwitterActivitySerializerTest.java | 8 +++- 5 files changed, 43 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/06fd10ef/streams-contrib/streams-provider-datasift/pom.xml ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/pom.xml b/streams-contrib/streams-provider-datasift/pom.xml index ae96d4e..0753325 100644 --- a/streams-contrib/streams-provider-datasift/pom.xml +++ b/streams-contrib/streams-provider-datasift/pom.xml @@ -145,36 +145,24 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <id>add-test-source</id> + <phase>generate-sources</phase> + <goals> + <goal>add-test-source</goal> + </goals> + <configuration> + <sources> + <source>src/test/java17</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> - <profiles> - <profile> - <id>java7</id> - <activation> - <jdk>[1.7,]</jdk> - </activation> - <build> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <executions> - <execution> - <id>add-test-source</id> - <phase>generate-sources</phase> - <goals> - <goal>add-test-source</goal> - </goals> - <configuration> - <sources> - <source>src/test/java17</source> - </sources> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> </project> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/06fd10ef/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java index 8f7ad43..dcc7ba9 100644 --- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java +++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftActivitySerializerTest.java @@ -13,7 +13,9 @@ import org.apache.streams.pojo.json.Actor; import org.junit.Before; import org.junit.Test; +import java.io.InputStream; import java.util.Scanner; +import java.util.regex.Pattern; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -24,6 +26,8 @@ public class DatasiftActivitySerializerTest { protected static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance(Lists.newArrayList(StreamsDatasiftMapper.DATASIFT_FORMAT)); + protected static Pattern newLinePattern = Pattern.compile("(\\r\\n?|\\n)", Pattern.MULTILINE); + @Before public void initSerializer() { SERIALIZER = new DatasiftActivitySerializer(); @@ -31,7 +35,10 @@ public class DatasiftActivitySerializerTest { @Test public void testConversion() throws Exception { - Scanner scanner = new Scanner(DatasiftActivitySerializerTest.class.getResourceAsStream("/rand_sample_datasift_json.txt")); + + InputStream testFileStream = DatasiftTwitterActivitySerializerTest.class.getResourceAsStream("/rand_sample_datasift_json.txt"); + Scanner scanner = new Scanner(testFileStream, "UTF-8").useDelimiter(newLinePattern); + String line = null; while(scanner.hasNextLine()) { try { http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/06fd10ef/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializerTest.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializerTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializerTest.java index 5350d74..c409642 100644 --- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializerTest.java +++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInstagramActivitySerializerTest.java @@ -12,6 +12,7 @@ import org.apache.streams.pojo.json.Actor; import org.junit.Before; import org.junit.Test; +import java.io.InputStream; import java.util.Scanner; import static org.junit.Assert.assertEquals; @@ -28,7 +29,10 @@ public class DatasiftInstagramActivitySerializerTest extends DatasiftActivitySer @Test @Override public void testConversion() throws Exception { - Scanner scanner = new Scanner(DatasiftActivitySerializerTest.class.getResourceAsStream("/instagram_datasift_json.txt")); + + InputStream testFileStream = DatasiftTwitterActivitySerializerTest.class.getResourceAsStream("/instagram_datasift_json.txt"); + Scanner scanner = new Scanner(testFileStream, "UTF-8").useDelimiter(newLinePattern); + String line = null; while(scanner.hasNextLine()) { line = scanner.nextLine(); http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/06fd10ef/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializerTest.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializerTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializerTest.java index 21d4ebb..c9886eb 100644 --- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializerTest.java +++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftInteractionActivitySerializerTest.java @@ -12,6 +12,7 @@ import org.apache.streams.pojo.json.Actor; import org.junit.Before; import org.junit.Test; +import java.io.InputStream; import java.util.Scanner; import static org.junit.Assert.assertEquals; @@ -28,7 +29,10 @@ public class DatasiftInteractionActivitySerializerTest extends DatasiftActivityS @Test @Override public void testConversion() throws Exception { - Scanner scanner = new Scanner(DatasiftInteractionActivitySerializerTest.class.getResourceAsStream("/rand_sample_datasift_json.txt")); + + InputStream testFileStream = DatasiftInteractionActivitySerializerTest.class.getResourceAsStream("/rand_sample_datasift_json.txt"); + Scanner scanner = new Scanner(testFileStream, "UTF-8").useDelimiter(newLinePattern); + String line = null; while(scanner.hasNextLine()) { try { http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/06fd10ef/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializerTest.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializerTest.java b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializerTest.java index 33b1f77..f458e21 100644 --- a/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializerTest.java +++ b/streams-contrib/streams-provider-datasift/src/test/java17/org/apache/streams/datasift/serializer/DatasiftTwitterActivitySerializerTest.java @@ -12,7 +12,10 @@ import org.apache.streams.pojo.json.Actor; import org.junit.Before; import org.junit.Test; +import java.io.InputStream; +import java.net.URL; import java.util.Scanner; +import java.util.regex.Pattern; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -28,7 +31,10 @@ public class DatasiftTwitterActivitySerializerTest extends DatasiftActivitySeria @Test @Override public void testConversion() throws Exception { - Scanner scanner = new Scanner(DatasiftTwitterActivitySerializerTest.class.getResourceAsStream("/twitter_datasift_json.txt")); + + InputStream testFileStream = DatasiftTwitterActivitySerializerTest.class.getResourceAsStream("/twitter_datasift_json.txt"); + Scanner scanner = new Scanner(testFileStream, "UTF-8").useDelimiter(newLinePattern); + String line = null; while(scanner.hasNextLine()) { line = scanner.nextLine();
