STREAMS-469: fit failing ITs in streams-provider-rss As best I can tell, junit forking was allowing the tests to pass - they started failing under TestNG. Wound up removing the static modifier from some state where it didnât really make sense. Along the way, upgraded the underlying library to one under active maintenance.
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/21086e9a Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/21086e9a Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/21086e9a Branch: refs/heads/master Commit: 21086e9a84721ef0d777a65a9cbd9e1f28040b18 Parents: 899bb84 Author: Steve Blackmon <sblack...@apache.org> Authored: Sun Dec 11 21:35:05 2016 -0600 Committer: Steve Blackmon <sblack...@apache.org> Committed: Sun Dec 11 21:35:05 2016 -0600 ---------------------------------------------------------------------- streams-contrib/streams-provider-rss/pom.xml | 4 +- .../streams/rss/provider/RssEventProcessor.java | 3 +- .../rss/provider/RssStreamProviderTask.java | 11 ++-- .../rss/serializer/SyndEntrySerializer.java | 39 +++++++------- .../rss/provider/RssStreamProviderTaskIT.java | 55 ++++++++------------ 5 files changed, 51 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/21086e9a/streams-contrib/streams-provider-rss/pom.xml ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-rss/pom.xml b/streams-contrib/streams-provider-rss/pom.xml index 5cf8b79..6a237d5 100644 --- a/streams-contrib/streams-provider-rss/pom.xml +++ b/streams-contrib/streams-provider-rss/pom.xml @@ -86,9 +86,9 @@ <scope>test</scope> </dependency> <dependency> - <groupId>rome</groupId> + <groupId>com.rometools</groupId> <artifactId>rome</artifactId> - <version>1.0</version> + <version>1.7.0</version> </dependency> <dependency> <groupId>org.mockito</groupId> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/21086e9a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssEventProcessor.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssEventProcessor.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssEventProcessor.java index 078356c..cf768e0 100644 --- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssEventProcessor.java +++ b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssEventProcessor.java @@ -24,7 +24,8 @@ import org.apache.streams.rss.serializer.SyndEntryActivitySerializer; import org.apache.streams.rss.serializer.SyndEntrySerializer; import com.fasterxml.jackson.databind.ObjectMapper; -import com.sun.syndication.feed.synd.SyndEntry; +import com.rometools.rome.feed.synd.SyndEntry; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/21086e9a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java index 03a66d1..8437ee0 100644 --- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java +++ b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/provider/RssStreamProviderTask.java @@ -25,10 +25,11 @@ import org.apache.streams.rss.serializer.SyndEntrySerializer; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; -import com.sun.syndication.feed.synd.SyndEntry; -import com.sun.syndication.feed.synd.SyndFeed; -import com.sun.syndication.io.FeedException; -import com.sun.syndication.io.SyndFeedInput; +import com.rometools.rome.feed.synd.SyndEntry; +import com.rometools.rome.feed.synd.SyndFeed; +import com.rometools.rome.io.FeedException; +import com.rometools.rome.io.SyndFeedInput; + import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,7 +78,7 @@ public class RssStreamProviderTask implements Runnable { * Map that contains the Set of previously seen articles by an rss feed. */ @VisibleForTesting - protected static final Map<String, Set<String>> PREVIOUSLY_SEEN = new ConcurrentHashMap<>(); + protected Map<String, Set<String>> PREVIOUSLY_SEEN = new ConcurrentHashMap<>(); private BlockingQueue<StreamsDatum> dataQueue; http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/21086e9a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntrySerializer.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntrySerializer.java b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntrySerializer.java index 63920e7..1b84d0a 100644 --- a/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntrySerializer.java +++ b/streams-contrib/streams-provider-rss/src/main/java/org/apache/streams/rss/serializer/SyndEntrySerializer.java @@ -21,16 +21,15 @@ package org.apache.streams.rss.serializer; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.sun.syndication.feed.module.Module; -import com.sun.syndication.feed.rss.Category; -import com.sun.syndication.feed.rss.Content; -import com.sun.syndication.feed.rss.Enclosure; -import com.sun.syndication.feed.synd.SyndContent; -import com.sun.syndication.feed.synd.SyndEnclosure; -import com.sun.syndication.feed.synd.SyndEntry; -import com.sun.syndication.feed.synd.SyndFeed; -import com.sun.syndication.feed.synd.SyndImage; -import com.sun.syndication.feed.synd.SyndLinkImpl; +import com.rometools.rome.feed.module.Module; +import com.rometools.rome.feed.rss.Enclosure; +import com.rometools.rome.feed.synd.SyndContent; +import com.rometools.rome.feed.synd.SyndEnclosure; +import com.rometools.rome.feed.synd.SyndEntry; +import com.rometools.rome.feed.synd.SyndFeed; +import com.rometools.rome.feed.synd.SyndImage; +import com.rometools.rome.feed.synd.SyndLinkImpl; + import org.joda.time.format.DateTimeFormatter; import org.joda.time.format.ISODateTimeFormat; import org.slf4j.Logger; @@ -82,9 +81,9 @@ public class SyndEntrySerializer { } ArrayNode cats = factory.arrayNode(); for (Object obj : categories) { - if (obj instanceof Category) { + if (obj instanceof com.rometools.rome.feed.rss.Category) { ObjectNode catNode = factory.objectNode(); - Category category = (Category) obj; + com.rometools.rome.feed.rss.Category category = (com.rometools.rome.feed.rss.Category) obj; if (category.getDomain() != null) { catNode.put("domain", category.getDomain()); } @@ -92,8 +91,8 @@ public class SyndEntrySerializer { catNode.put("value", category.getValue()); } cats.add(catNode); - } else if (obj instanceof com.sun.syndication.feed.atom.Category) { - com.sun.syndication.feed.atom.Category category = (com.sun.syndication.feed.atom.Category) obj; + } else if (obj instanceof com.rometools.rome.feed.atom.Category) { + com.rometools.rome.feed.atom.Category category = (com.rometools.rome.feed.atom.Category) obj; ObjectNode catNode = factory.objectNode(); if (category.getLabel() != null) { catNode.put("label", category.getLabel()); @@ -120,13 +119,13 @@ public class SyndEntrySerializer { ArrayNode contentsArray = factory.arrayNode(); for (Object obj : contents) { ObjectNode content = factory.objectNode(); - if (obj instanceof Content) { - Content rssContent = (Content) obj; + if (obj instanceof com.rometools.rome.feed.rss.Content) { + com.rometools.rome.feed.rss.Content rssContent = (com.rometools.rome.feed.rss.Content) obj; content.put("type", rssContent.getType()); content.put("value", rssContent.getValue()); } - if (obj instanceof com.sun.syndication.feed.atom.Content) { - com.sun.syndication.feed.atom.Content atomContent = (com.sun.syndication.feed.atom.Content) obj; + if (obj instanceof com.rometools.rome.feed.atom.Content) { + com.rometools.rome.feed.atom.Content atomContent = (com.rometools.rome.feed.atom.Content) obj; content.put("type", atomContent.getType()); content.put("value", atomContent.getValue()); content.put("mode", atomContent.getMode()); @@ -248,7 +247,9 @@ public class SyndEntrySerializer { } private void serializeLinks(ObjectNode root, JsonNodeFactory factory, List links) { - if (links.get(0) instanceof String) { + if( links.size() == 0 ) { + root.put("links", factory.arrayNode()); + } else if (links.get(0) instanceof String) { serializeListOfStrings(links, "links", root, factory); } else if (links.get(0) instanceof SyndLinkImpl) { ArrayNode linksArray = factory.arrayNode(); http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/21086e9a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskIT.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskIT.java b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskIT.java index 87ff8fc..564e215 100644 --- a/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskIT.java +++ b/streams-contrib/streams-provider-rss/src/test/java/org/apache/streams/rss/provider/RssStreamProviderTaskIT.java @@ -22,7 +22,6 @@ import org.apache.streams.core.StreamsDatum; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import java.net.URL; @@ -39,18 +38,6 @@ import static org.testng.Assert.assertTrue; public class RssStreamProviderTaskIT { /** - * clearPreviouslySeen. - */ - @BeforeClass - public void clearPreviouslySeen() { - //some test runners run in parallel so needs to be synchronized - //if tests are run in parallel test will have undetermined results. - synchronized (RssStreamProviderTask.PREVIOUSLY_SEEN) { - RssStreamProviderTask.PREVIOUSLY_SEEN.clear(); - } - } - - /** * Test that a task can read a valid rss from a url and queue the data. * @throws Exception Exception */ @@ -61,7 +48,7 @@ public class RssStreamProviderTaskIT { RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url"); Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml")); assertEquals(batch.size(), queue.size(), "Expected batch size to be the same as amount of queued datums"); - RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch); + task.PREVIOUSLY_SEEN.put("fake url", batch); //Test that it will out previously seen articles queue.clear(); batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml")); @@ -76,22 +63,22 @@ public class RssStreamProviderTaskIT { public void testPerpetualNoTimeFramePull() throws Exception { com.healthmarketscience.common.util.resource.Handler.init(); BlockingQueue<StreamsDatum> queue = new LinkedBlockingQueue<>(); - RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url", new DateTime().minusYears(5), 10000, true); + RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url", new DateTime().minusYears(10), 10000, true); Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml")); assertEquals(batch.size(), queue.size(), "Expected batch size to be the same as amount of queued datums"); - RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch); + task.PREVIOUSLY_SEEN.put("fake url", batch); //Test that it will not out previously seen articles queue.clear(); batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml")); - assertEquals(0, queue.size(), "Expected queue size to be 0"); - assertEquals(20, batch.size(), "Expected batch size to be 20"); - RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch); + assertEquals( queue.size(), 0 ); + assertEquals( batch.size(), 20 ); + task.PREVIOUSLY_SEEN.put("fake url", batch); //Test that not seen urls aren't blocked. queue.clear(); batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist2.xml")); assertEquals(batch.size(), queue.size()); - assertEquals(25, queue.size(), "Expected queue size to be 25"); - assertEquals(25, batch.size(), "Expected batch size to be 25"); + assertEquals( queue.size(), 25); + assertEquals( batch.size(), 25); } /** @@ -105,15 +92,15 @@ public class RssStreamProviderTaskIT { DateTime publishedSince = new DateTime().withYear(2014).withDayOfMonth(5).withMonthOfYear(9).withZone(DateTimeZone.UTC); RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url", publishedSince, 10000, false); Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml")); - assertEquals( 15, queue.size()); - assertEquals( 20 , batch.size()); + assertEquals( queue.size(), 15); + assertEquals( batch.size(), 20); assertTrue( queue.size() < batch.size()); - RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch); + task.PREVIOUSLY_SEEN.put("fake url", batch); //Test that it will out previously seen articles queue.clear(); batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml")); - assertEquals( 15, queue.size()); - assertEquals( 20 , batch.size()); + assertEquals( queue.size(), 15); + assertEquals( batch.size(), 20); assertTrue( queue.size() < batch.size()); } @@ -128,22 +115,22 @@ public class RssStreamProviderTaskIT { DateTime publishedSince = new DateTime().withYear(2014).withDayOfMonth(5).withMonthOfYear(9).withZone(DateTimeZone.UTC); RssStreamProviderTask task = new RssStreamProviderTask(queue, "fake url", publishedSince, 10000, true); Set<String> batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml")); - assertEquals( 15, queue.size()); - assertEquals( 20 , batch.size()); + assertEquals( queue.size(),15); + assertEquals( batch.size(), 20); assertTrue( queue.size() < batch.size()); - RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch); + task.PREVIOUSLY_SEEN.put("fake url", batch); //Test that it will not out put previously seen articles queue.clear(); batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist1.xml")); - assertEquals( 0, queue.size()); - assertEquals( 20 , batch.size()); + assertEquals( queue.size(), 0); + assertEquals( batch.size(), 20); assertTrue( queue.size() < batch.size()); - RssStreamProviderTask.PREVIOUSLY_SEEN.put("fake url", batch); + task.PREVIOUSLY_SEEN.put("fake url", batch); batch = task.queueFeedEntries(new URL("resource:///test_rss_xml/economist2.xml")); assertTrue( queue.size() < batch.size()); - assertEquals(3, queue.size(), "Expected queue size to be 0"); - assertEquals(25, batch.size(), "Expected batch size to be 0"); + assertEquals(queue.size(), 3); + assertEquals(batch.size(), 25); }
