http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/recentmedia/InstagramRecentMediaProvider.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/recentmedia/InstagramRecentMediaProvider.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/recentmedia/InstagramRecentMediaProvider.java index 9a31b5a..c68ef95 100644 --- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/recentmedia/InstagramRecentMediaProvider.java +++ b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/recentmedia/InstagramRecentMediaProvider.java @@ -12,15 +12,9 @@ software distributed under the License is distributed on an KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package org.apache.streams.instagram.provider.recentmedia; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.Uninterruptibles; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; -import com.typesafe.config.ConfigParseOptions; import org.apache.streams.config.ComponentConfigurator; import org.apache.streams.config.StreamsConfiguration; import org.apache.streams.config.StreamsConfigurator; @@ -29,6 +23,14 @@ import org.apache.streams.instagram.InstagramConfiguration; import org.apache.streams.instagram.provider.InstagramAbstractProvider; import org.apache.streams.instagram.provider.InstagramDataCollector; import org.apache.streams.jackson.StreamsJacksonMapper; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.Uninterruptibles; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigParseOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,78 +42,91 @@ import java.util.Iterator; import java.util.concurrent.TimeUnit; /** - * Instagram {@link org.apache.streams.core.StreamsProvider} that provides the recent media data for a group of users + * Instagram {@link org.apache.streams.core.StreamsProvider} that provides the recent media data for a group of users. * + * <p/> * Retrieve recent posts from a list of user ids or names. * - * To use from command line: - * - * Supply (at least) the following required configuration in application.conf: - * - * instagram.clientKey - * instagram.usersInfo.authorizedTokens - * instagram.usersInfo.users - * - * Launch using: - * - * mvn exec:java -Dexec.mainClass=org.apache.streams.instagram.provider.recentmedia.InstagramRecentMediaProvider -Dexec.args="application.conf media.json" */ public class InstagramRecentMediaProvider extends InstagramAbstractProvider { - public static final String STREAMS_ID = "InstagramRecentMediaProvider"; - - private static final Logger LOGGER = LoggerFactory.getLogger(InstagramRecentMediaProvider.class); - - private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance(); - - public InstagramRecentMediaProvider() { - } - - public InstagramRecentMediaProvider(InstagramConfiguration config) { - super(config); - } - - @Override - protected InstagramDataCollector getInstagramDataCollector() { - return new InstagramRecentMediaCollector(super.dataQueue, super.config); - } - - public static void main(String[] args) throws Exception { - - Preconditions.checkArgument(args.length >= 2); - - String configfile = args[0]; - String outfile = args[1]; - - Config reference = ConfigFactory.load(); - File conf_file = new File(configfile); - assert(conf_file.exists()); - Config conf = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - - Config typesafe = conf.withFallback(reference).resolve(); - - StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe); - InstagramConfiguration config = new ComponentConfigurator<>(InstagramConfiguration.class).detectConfiguration(typesafe, "instagram"); - InstagramRecentMediaProvider provider = new InstagramRecentMediaProvider(config); - - PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile))); - provider.prepare(config); - provider.startStream(); - do { - Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS); - Iterator<StreamsDatum> iterator = provider.readCurrent().iterator(); - while(iterator.hasNext()) { - StreamsDatum datum = iterator.next(); - String json; - try { - json = MAPPER.writeValueAsString(datum.getDocument()); - outStream.println(json); - } catch (JsonProcessingException e) { - System.err.println(e.getMessage()); - } - } - } while( provider.isRunning()); - provider.cleanUp(); - outStream.flush(); + public static final String STREAMS_ID = "InstagramRecentMediaProvider"; + + private static final Logger LOGGER = LoggerFactory.getLogger(InstagramRecentMediaProvider.class); + + private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance(); + + public InstagramRecentMediaProvider() { + } + + public InstagramRecentMediaProvider(InstagramConfiguration config) { + super(config); + } + + @Override + protected InstagramDataCollector getInstagramDataCollector() { + return new InstagramRecentMediaCollector(super.dataQueue, super.config); + } + + /** + * To use from command line: + * + * <p/> + * Supply (at least) the following required configuration in application.conf: + * + * <p/> + * instagram.clientKey + * instagram.usersInfo.authorizedTokens + * instagram.usersInfo.users + * + * <p/> + * Launch using: + * + * <p/> + * mvn exec:java \ + * -Dexec.mainClass=org.apache.streams.instagram.provider.recentmedia.InstagramRecentMediaProvider \ + * -Dexec.args="application.conf media.json" + * + * @param args args + * @throws Exception Exception + */ + public static void main(String[] args) throws Exception { + + Preconditions.checkArgument(args.length >= 2); + + String configfile = args[0]; + String outfile = args[1]; + + Config reference = ConfigFactory.load(); + File file = new File(configfile); + assert (file.exists()); + Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false)); + + Config typesafe = conf.withFallback(reference).resolve(); + + StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe); + InstagramConfiguration config = new ComponentConfigurator<>(InstagramConfiguration.class).detectConfiguration(typesafe, "instagram"); + InstagramRecentMediaProvider provider = new InstagramRecentMediaProvider(config); + + PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile))); + provider.prepare(config); + provider.startStream(); + do { + Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS); + Iterator<StreamsDatum> iterator = provider.readCurrent().iterator(); + while (iterator.hasNext()) { + StreamsDatum datum = iterator.next(); + String json; + try { + json = MAPPER.writeValueAsString(datum.getDocument()); + outStream.println(json); + } catch (JsonProcessingException ex) { + System.err.println(ex.getMessage()); + } + } } + while ( provider.isRunning()); + provider.cleanUp(); + outStream.flush(); + } }
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoCollector.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoCollector.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoCollector.java index 0985ae8..98d0f3c 100644 --- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoCollector.java +++ b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoCollector.java @@ -12,13 +12,15 @@ software distributed under the License is distributed on an KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package org.apache.streams.instagram.provider.userinfo; -import com.google.common.collect.Lists; import org.apache.streams.core.StreamsDatum; import org.apache.streams.instagram.InstagramConfiguration; import org.apache.streams.instagram.User; import org.apache.streams.instagram.provider.InstagramDataCollector; + +import com.google.common.collect.Lists; import org.jinstagram.entity.users.basicinfo.UserInfo; import org.jinstagram.entity.users.basicinfo.UserInfoData; import org.jinstagram.exceptions.InstagramBadRequestException; @@ -33,60 +35,60 @@ import java.util.Queue; * InstagramDataCollector that pulls UserInfoData from Instagram * @see org.apache.streams.instagram.provider.InstagramDataCollector */ -public class InstagramUserInfoCollector extends InstagramDataCollector<UserInfoData>{ +public class InstagramUserInfoCollector extends InstagramDataCollector<UserInfoData> { - private static final Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoCollector.class); - protected static final int MAX_ATTEMPTS = 5; + private static final Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoCollector.class); + protected static final int MAX_ATTEMPTS = 5; - private int consecutiveErrorCount; + private int consecutiveErrorCount; - public InstagramUserInfoCollector(Queue<StreamsDatum> dataQueue, InstagramConfiguration config) { - super(dataQueue, config); - this.consecutiveErrorCount = 0; - } + public InstagramUserInfoCollector(Queue<StreamsDatum> dataQueue, InstagramConfiguration config) { + super(dataQueue, config); + this.consecutiveErrorCount = 0; + } - @Override - protected void collectInstagramDataForUser(User user) throws Exception { - int attempt = 0; - boolean successful = false; - UserInfo userInfo = null; - while(!successful && attempt < MAX_ATTEMPTS) { - ++attempt; - try { - userInfo = getNextInstagramClient().getUserInfo(user.getUserId()); - } catch (Exception e) { - if(e instanceof InstagramRateLimitException) { - LOGGER.warn("Hit rate limit exception, backing off."); - super.backOffStrategy.backOff(); - } else if(e instanceof InstagramBadRequestException) { - LOGGER.error("Sent a bad request to Instagram, skipping user : {}", user.getUserId()); - attempt = MAX_ATTEMPTS; - ++this.consecutiveErrorCount; - } else { - LOGGER.error("Expection while polling instagram : {}", e); - ++this.consecutiveErrorCount; - } - if(this.consecutiveErrorCount >= Math.max(super.numAvailableTokens(), MAX_ATTEMPTS * 2)) { - LOGGER.error("Consecutive Errors above acceptable limits, ending collection of data."); - throw new Exception("Consecutive Errors above acceptable limits : "+this.consecutiveErrorCount); - } - } - if(successful = (userInfo != null)) { - this.consecutiveErrorCount = 0; - List<UserInfoData> data = Lists.newLinkedList(); - data.add(userInfo.getData()); - super.queueData(data, user.getUserId()); - } + @Override + protected void collectInstagramDataForUser(User user) throws Exception { + int attempt = 0; + boolean successful = false; + UserInfo userInfo = null; + while (!successful && attempt < MAX_ATTEMPTS) { + ++attempt; + try { + userInfo = getNextInstagramClient().getUserInfo(user.getUserId()); + } catch (Exception ex) { + if (ex instanceof InstagramRateLimitException) { + LOGGER.warn("Hit rate limit exception, backing off."); + super.backOffStrategy.backOff(); + } else if (ex instanceof InstagramBadRequestException) { + LOGGER.error("Sent a bad request to Instagram, skipping user : {}", user.getUserId()); + attempt = MAX_ATTEMPTS; + ++this.consecutiveErrorCount; + } else { + LOGGER.error("Expection while polling instagram : {}", ex); + ++this.consecutiveErrorCount; } - if(attempt == MAX_ATTEMPTS) { - LOGGER.error("Failed to collect data for user : {}", user.getUserId()); + if (this.consecutiveErrorCount >= Math.max(super.numAvailableTokens(), MAX_ATTEMPTS * 2)) { + LOGGER.error("Consecutive Errors above acceptable limits, ending collection of data."); + throw new Exception("Consecutive Errors above acceptable limits : " + this.consecutiveErrorCount); } + } + if (successful = (userInfo != null)) { + this.consecutiveErrorCount = 0; + List<UserInfoData> data = Lists.newLinkedList(); + data.add(userInfo.getData()); + super.queueData(data, user.getUserId()); + } } - - @Override - protected StreamsDatum convertToStreamsDatum(UserInfoData item) { - return new StreamsDatum(item, item.getId()); + if (attempt == MAX_ATTEMPTS) { + LOGGER.error("Failed to collect data for user : {}", user.getUserId()); } + } + + @Override + protected StreamsDatum convertToStreamsDatum(UserInfoData item) { + return new StreamsDatum(item, item.getId()); + } } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoProvider.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoProvider.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoProvider.java index 0a47944..4469cd6 100644 --- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoProvider.java +++ b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/provider/userinfo/InstagramUserInfoProvider.java @@ -12,16 +12,9 @@ software distributed under the License is distributed on an KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package org.apache.streams.instagram.provider.userinfo; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.Uninterruptibles; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; -import com.typesafe.config.ConfigParseOptions; import org.apache.streams.config.ComponentConfigurator; import org.apache.streams.config.StreamsConfiguration; import org.apache.streams.config.StreamsConfigurator; @@ -31,6 +24,14 @@ import org.apache.streams.instagram.InstagramUserInformationConfiguration; import org.apache.streams.instagram.provider.InstagramAbstractProvider; import org.apache.streams.instagram.provider.InstagramDataCollector; import org.apache.streams.jackson.StreamsJacksonMapper; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.Uninterruptibles; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigParseOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,80 +43,92 @@ import java.util.Iterator; import java.util.concurrent.TimeUnit; /** - * Instagram provider that pulls UserInfoData from Instagram - * @see org.apache.streams.instagram.provider.InstagramAbstractProvider + * Instagram provider that pulls UserInfoData from Instagram. * + * <p/> * Retrieve latest user details from a list of user ids or names. - * - * To use from command line: - * - * Supply (at least) the following required configuration in application.conf: - * - * instagram.clientKey - * instagram.usersInfo.authorizedTokens - * instagram.usersInfo.users - * - * Launch using: - * - * mvn exec:java -Dexec.mainClass=org.apache.streams.instagram.provider.userinfo.InstagramUserInfoProvider -Dexec.args="application.conf userinfo.json" */ public class InstagramUserInfoProvider extends InstagramAbstractProvider { - public static final String STREAMS_ID = "InstagramUserInfoProvider"; - - private static final Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoProvider.class); - - private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance(); - - public InstagramUserInfoProvider() { - super(); - } - - public InstagramUserInfoProvider(InstagramConfiguration config) { - super(config); - } - - @Override - protected InstagramDataCollector getInstagramDataCollector() { - return new InstagramUserInfoCollector(super.dataQueue, super.config); - } - - public static void main(String[] args) throws Exception { - - Preconditions.checkArgument(args.length >= 2); - - String configfile = args[0]; - String outfile = args[1]; - - Config reference = ConfigFactory.load(); - File conf_file = new File(configfile); - assert(conf_file.exists()); - Config conf = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - - Config typesafe = conf.withFallback(reference).resolve(); - - StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe); - InstagramUserInformationConfiguration config = new ComponentConfigurator<>(InstagramUserInformationConfiguration.class).detectConfiguration(typesafe, "instagram"); - InstagramUserInfoProvider provider = new InstagramUserInfoProvider(config); - - PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile))); - provider.prepare(config); - provider.startStream(); - do { - Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS); - Iterator<StreamsDatum> iterator = provider.readCurrent().iterator(); - while(iterator.hasNext()) { - StreamsDatum datum = iterator.next(); - String json; - try { - json = MAPPER.writeValueAsString(datum.getDocument()); - outStream.println(json); - } catch (JsonProcessingException e) { - System.err.println(e.getMessage()); - } - } - } while( provider.isRunning()); - provider.cleanUp(); - outStream.flush(); + public static final String STREAMS_ID = "InstagramUserInfoProvider"; + + private static final Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoProvider.class); + + private static ObjectMapper MAPPER = StreamsJacksonMapper.getInstance(); + + public InstagramUserInfoProvider() { + super(); + } + + public InstagramUserInfoProvider(InstagramConfiguration config) { + super(config); + } + + @Override + protected InstagramDataCollector getInstagramDataCollector() { + return new InstagramUserInfoCollector(super.dataQueue, super.config); + } + + /** + * To use from command line: + * + * <p/> + * Supply (at least) the following required configuration in application.conf: + * + * <p/> + * instagram.clientKey + * instagram.usersInfo.authorizedTokens + * instagram.usersInfo.users + * + * <p/> + * Launch using: + * + * <p/> + * mvn exec:java \ + * -Dexec.mainClass=org.apache.streams.instagram.provider.userinfo.InstagramUserInfoProvider \ + * -Dexec.args="application.conf userinfo.json" + * + * @param args args + * @throws Exception Exception + */ + public static void main(String[] args) throws Exception { + + Preconditions.checkArgument(args.length >= 2); + + String configfile = args[0]; + String outfile = args[1]; + + Config reference = ConfigFactory.load(); + File file = new File(configfile); + assert (file.exists()); + Config conf = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false)); + + Config typesafe = conf.withFallback(reference).resolve(); + + StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe); + InstagramUserInformationConfiguration config = new ComponentConfigurator<>(InstagramUserInformationConfiguration.class) + .detectConfiguration(typesafe, "instagram"); + InstagramUserInfoProvider provider = new InstagramUserInfoProvider(config); + + PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile))); + provider.prepare(config); + provider.startStream(); + do { + Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS); + Iterator<StreamsDatum> iterator = provider.readCurrent().iterator(); + while (iterator.hasNext()) { + StreamsDatum datum = iterator.next(); + String json; + try { + json = MAPPER.writeValueAsString(datum.getDocument()); + outStream.println(json); + } catch (JsonProcessingException ex) { + System.err.println(ex.getMessage()); + } + } } + while ( provider.isRunning()); + provider.cleanUp(); + outStream.flush(); + } } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramMediaFeedDataConverter.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramMediaFeedDataConverter.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramMediaFeedDataConverter.java index 8795e2c..c9bbd85 100644 --- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramMediaFeedDataConverter.java +++ b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramMediaFeedDataConverter.java @@ -18,11 +18,12 @@ package org.apache.streams.instagram.serializer; -import com.google.common.collect.Lists; -import org.apache.commons.lang.NotImplementedException; import org.apache.streams.data.ActivityConverter; import org.apache.streams.exceptions.ActivityConversionException; import org.apache.streams.pojo.json.Activity; + +import com.google.common.collect.Lists; +import org.apache.commons.lang.NotImplementedException; import org.jinstagram.entity.users.feed.MediaFeedData; import java.io.Serializable; @@ -30,48 +31,47 @@ import java.util.List; import static org.apache.streams.instagram.serializer.util.InstagramActivityUtil.updateActivity; -public class InstagramMediaFeedDataConverter implements ActivityConverter<MediaFeedData>, Serializable -{ +public class InstagramMediaFeedDataConverter implements ActivityConverter<MediaFeedData>, Serializable { - public static Class requiredClass = MediaFeedData.class; + public static Class requiredClass = MediaFeedData.class; - public InstagramMediaFeedDataConverter() { + public InstagramMediaFeedDataConverter() { - } + } - @Override - public Class requiredClass() { - return requiredClass; - } + @Override + public Class requiredClass() { + return requiredClass; + } - @Override - public String serializationFormat() { - return null; - } + @Override + public String serializationFormat() { + return null; + } - @Override - public MediaFeedData fromActivity(Activity deserialized) throws ActivityConversionException { - throw new NotImplementedException(); - } + @Override + public MediaFeedData fromActivity(Activity deserialized) throws ActivityConversionException { + throw new NotImplementedException(); + } - @Override - public List<Activity> toActivityList(MediaFeedData item) throws ActivityConversionException { + @Override + public List<MediaFeedData> fromActivityList(List<Activity> list) throws ActivityConversionException { + throw new NotImplementedException(); + } - Activity activity = new Activity(); + @Override + public List<Activity> toActivityList(MediaFeedData item) throws ActivityConversionException { - updateActivity(item, activity); + Activity activity = new Activity(); - return Lists.newArrayList(activity); - } + updateActivity(item, activity); - @Override - public List<MediaFeedData> fromActivityList(List<Activity> list) throws ActivityConversionException { - throw new NotImplementedException(); - } + return Lists.newArrayList(activity); + } - @Override - public List<Activity> toActivityList(List<MediaFeedData> list) throws ActivityConversionException { - throw new NotImplementedException(); - } + @Override + public List<Activity> toActivityList(List<MediaFeedData> list) throws ActivityConversionException { + throw new NotImplementedException(); + } } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramUserInfoDataConverter.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramUserInfoDataConverter.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramUserInfoDataConverter.java index 4fa680c..1109163 100644 --- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramUserInfoDataConverter.java +++ b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/InstagramUserInfoDataConverter.java @@ -23,6 +23,7 @@ import org.apache.streams.exceptions.ActivityConversionException; import org.apache.streams.pojo.json.ActivityObject; import org.apache.streams.pojo.json.Image; import org.apache.streams.pojo.json.Provider; + import org.jinstagram.entity.users.basicinfo.UserInfoData; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -33,56 +34,56 @@ import java.util.HashMap; import java.util.Map; /** - * + * InstagramUserInfoDataConverter */ public class InstagramUserInfoDataConverter implements ActivityObjectConverter<UserInfoData> { - private static final Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoDataConverter.class); + private static final Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoDataConverter.class); - private static final String STREAMS_ID_PREFIX = "id:instagram:"; - private static final String PROVIDER_ID = "id:provider:instagram"; - private static final String DISPLAY_NAME = "Instagram"; + private static final String STREAMS_ID_PREFIX = "id:instagram:"; + private static final String PROVIDER_ID = "id:provider:instagram"; + private static final String DISPLAY_NAME = "Instagram"; - @Override - public Class requiredClass() { - return UserInfoData.class; - } + @Override + public Class requiredClass() { + return UserInfoData.class; + } - @Override - public String serializationFormat() { - return null; - } + @Override + public String serializationFormat() { + return null; + } - @Override - public UserInfoData fromActivityObject(ActivityObject deserialized) throws ActivityConversionException { - return null; - } + @Override + public UserInfoData fromActivityObject(ActivityObject deserialized) throws ActivityConversionException { + return null; + } - @Override - public ActivityObject toActivityObject(UserInfoData serialized) throws ActivityConversionException { - ActivityObject activityObject = new ActivityObject(); - activityObject.setObjectType("page"); - Provider provider = new Provider(); - provider.setId(PROVIDER_ID); - provider.setDisplayName(DISPLAY_NAME); - activityObject.getAdditionalProperties().put("provider", provider); - activityObject.setPublished(DateTime.now().withZone(DateTimeZone.UTC)); - Image image = new Image(); - image.setUrl(serialized.getProfilePicture()); - activityObject.setImage(image); - activityObject.setId(STREAMS_ID_PREFIX+serialized.getId()); - activityObject.setSummary(serialized.getBio()); - activityObject.setAdditionalProperty("handle", serialized.getUsername()); - activityObject.setDisplayName(serialized.getFullName()); - activityObject.setUrl(serialized.getWebsite()); - Map<String, Object> extensions = new HashMap<>(); - activityObject.setAdditionalProperty("extensions", extensions); - extensions.put("screenName", serialized.getUsername()); - extensions.put("posts", serialized.getCounts().getMedia()); - extensions.put("followers", serialized.getCounts().getFollowedBy()); - extensions.put("website", serialized.getWebsite()); - extensions.put("following", serialized.getCounts().getFollows()); - return activityObject; - } + @Override + public ActivityObject toActivityObject(UserInfoData serialized) throws ActivityConversionException { + ActivityObject activityObject = new ActivityObject(); + activityObject.setObjectType("page"); + Provider provider = new Provider(); + provider.setId(PROVIDER_ID); + provider.setDisplayName(DISPLAY_NAME); + activityObject.getAdditionalProperties().put("provider", provider); + activityObject.setPublished(DateTime.now().withZone(DateTimeZone.UTC)); + Image image = new Image(); + image.setUrl(serialized.getProfilePicture()); + activityObject.setImage(image); + activityObject.setId(STREAMS_ID_PREFIX + serialized.getId()); + activityObject.setSummary(serialized.getBio()); + activityObject.setAdditionalProperty("handle", serialized.getUsername()); + activityObject.setDisplayName(serialized.getFullName()); + activityObject.setUrl(serialized.getWebsite()); + Map<String, Object> extensions = new HashMap<>(); + activityObject.setAdditionalProperty("extensions", extensions); + extensions.put("screenName", serialized.getUsername()); + extensions.put("posts", serialized.getCounts().getMedia()); + extensions.put("followers", serialized.getCounts().getFollowedBy()); + extensions.put("website", serialized.getWebsite()); + extensions.put("following", serialized.getCounts().getFollows()); + return activityObject; + } } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java index 1c82da4..dd181de 100644 --- a/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java +++ b/streams-contrib/streams-provider-instagram/src/main/java/org/apache/streams/instagram/serializer/util/InstagramActivityUtil.java @@ -19,9 +19,6 @@ package org.apache.streams.instagram.serializer.util; -import com.google.common.base.Joiner; -import com.google.common.base.Optional; -import com.google.common.collect.Lists; import org.apache.streams.exceptions.ActivityConversionException; import org.apache.streams.exceptions.ActivitySerializerException; import org.apache.streams.pojo.extensions.ExtensionUtil; @@ -29,6 +26,10 @@ import org.apache.streams.pojo.json.Activity; import org.apache.streams.pojo.json.ActivityObject; import org.apache.streams.pojo.json.Image; import org.apache.streams.pojo.json.Provider; + +import com.google.common.base.Joiner; +import com.google.common.base.Optional; +import com.google.common.collect.Lists; import org.jinstagram.entity.comments.CommentData; import org.jinstagram.entity.common.Comments; import org.jinstagram.entity.common.ImageData; @@ -48,301 +49,305 @@ import java.util.List; import java.util.Map; /** - * Provides utilities for working with Activity objects within the context of Instagram + * Provides utilities for working with Activity objects within the context of Instagram. */ public class InstagramActivityUtil { - private static final Logger LOGGER = LoggerFactory.getLogger(InstagramActivityUtil.class); - /** - * Updates the given Activity object with the values from the item - * @param item the object to use as the source - * @param activity the target of the updates. Will receive all values from the tweet. - * @throws ActivityConversionException - */ - public static void updateActivity(MediaFeedData item, Activity activity) throws ActivityConversionException { - activity.setActor(buildActor(item)); - activity.setVerb("post"); - - if(item.getCreatedTime() != null) - activity.setPublished(new DateTime(Long.parseLong(item.getCreatedTime()) * 1000)); - - activity.setId(formatId(activity.getVerb(), - Optional.fromNullable( - item.getId()) - .orNull())); - - activity.setProvider(getProvider()); - activity.setUrl(item.getLink()); - activity.setObject(buildActivityObject(item)); - - if(item.getCaption() != null) - activity.setContent(item.getCaption().getText()); - - addInstagramExtensions(activity, item); - } - - /** - * Updates the given Activity object with the values from the item - * @param item the object to use as the source - * @param activity the target of the updates. Will receive all values from the tweet. - * @throws ActivitySerializerException - */ - public static void updateActivity(UserInfoData item, Activity activity) throws ActivitySerializerException { - activity.setActor(buildActor(item)); - activity.setId(null); - activity.setProvider(getProvider()); - } - - /** - * Builds an Actor object given a UserInfoData object - * @param item - * @return Actor object - */ - public static ActivityObject buildActor(UserInfoData item) { - ActivityObject actor = new ActivityObject(); - try { - Image image = new Image(); - image.setUrl(item.getProfilePicture()); + private static final Logger LOGGER = LoggerFactory.getLogger(InstagramActivityUtil.class); + /** + * Updates the given Activity object with the values from the item + * @param item the object to use as the source + * @param activity the target of the updates. Will receive all values from the tweet. + * @throws ActivityConversionException ActivityConversionException + */ - Counts counts = item.getCounts(); + public static void updateActivity(MediaFeedData item, Activity activity) throws ActivityConversionException { + activity.setActor(buildActor(item)); + activity.setVerb("post"); - Map<String, Object> extensions = new HashMap<>(); - - extensions.put("followers", counts.getFollowedBy()); - extensions.put("follows", counts.getFollows()); - extensions.put("screenName", item.getUsername()); - extensions.put("posts", counts.getMedia()); + if (item.getCreatedTime() != null) { + activity.setPublished(new DateTime(Long.parseLong(item.getCreatedTime()) * 1000)); + } - actor.setId(formatId(String.valueOf(item.getId()))); - actor.setImage(image); - actor.setDisplayName(item.getFullName()); - actor.setSummary(item.getBio()); - actor.setUrl(item.getWebsite()); + activity.setId(formatId(activity.getVerb(), + Optional.fromNullable( + item.getId()) + .orNull())); - actor.setAdditionalProperty("handle", item.getUsername()); - actor.setAdditionalProperty("extensions", extensions); - } catch (Exception e) { - LOGGER.error("Exception trying to build actor object: {}", e.getMessage()); - } + activity.setProvider(getProvider()); + activity.setUrl(item.getLink()); + activity.setObject(buildActivityObject(item)); - return actor; + if (item.getCaption() != null) { + activity.setContent(item.getCaption().getText()); } - /** - * Builds the actor - * @param item the item - * @return a valid Actor - */ - public static ActivityObject buildActor(MediaFeedData item) { - ActivityObject actor = new ActivityObject(); - - try { - Image image = new Image(); - image.setUrl(item.getUser().getProfilePictureUrl()); - - Map<String, Object> extensions = new HashMap<>(); - extensions.put("screenName", item.getUser().getUserName()); - - actor.setDisplayName(item.getUser().getFullName()); - actor.setSummary(item.getUser().getBio()); - actor.setUrl(item.getUser().getWebsiteUrl()); - - actor.setId(formatId(String.valueOf(item.getUser().getId()))); - actor.setImage(image); - actor.setAdditionalProperty("extensions", extensions); - actor.setAdditionalProperty("handle", item.getUser().getUserName()); - } catch (Exception e) { - LOGGER.error("Exception trying to build actor object: {}", e.getMessage()); - } - - return actor; + addInstagramExtensions(activity, item); + } + + /** + * Updates the given Activity object with the values from the item + * @param item the object to use as the source + * @param activity the target of the updates. Will receive all values from the tweet. + * @throws ActivitySerializerException ActivitySerializerException + */ + public static void updateActivity(UserInfoData item, Activity activity) throws ActivitySerializerException { + activity.setActor(buildActor(item)); + activity.setId(null); + activity.setProvider(getProvider()); + } + + /** + * Builds an Actor object given a UserInfoData object. + * @param item UserInfoData item + * @return Actor object + */ + public static ActivityObject buildActor(UserInfoData item) { + ActivityObject actor = new ActivityObject(); + + try { + Image image = new Image(); + image.setUrl(item.getProfilePicture()); + + Counts counts = item.getCounts(); + + Map<String, Object> extensions = new HashMap<>(); + + extensions.put("followers", counts.getFollowedBy()); + extensions.put("follows", counts.getFollows()); + extensions.put("screenName", item.getUsername()); + extensions.put("posts", counts.getMedia()); + + actor.setId(formatId(String.valueOf(item.getId()))); + actor.setImage(image); + actor.setDisplayName(item.getFullName()); + actor.setSummary(item.getBio()); + actor.setUrl(item.getWebsite()); + + actor.setAdditionalProperty("handle", item.getUsername()); + actor.setAdditionalProperty("extensions", extensions); + } catch (Exception ex) { + LOGGER.error("Exception trying to build actor object: {}", ex.getMessage()); } - /** - * Builds the ActivityObject - * @param item the item - * @return a valid Activity Object - */ - public static ActivityObject buildActivityObject(MediaFeedData item) { - ActivityObject actObj = new ActivityObject(); - - actObj.setObjectType(item.getType()); - actObj.setAttachments(buildActivityObjectAttachments(item)); - - Image standardResolution = new Image(); - if(item.getType().equals("image") && item.getImages() != null) { - ImageData standardResolutionData = item.getImages().getStandardResolution(); - standardResolution.setHeight((long) standardResolutionData.getImageHeight()); - standardResolution.setWidth((long) standardResolutionData.getImageWidth()); - standardResolution.setUrl(standardResolutionData.getImageUrl()); - } else if(item.getType().equals("video") && item.getVideos() != null) { - VideoData standardResolutionData = item.getVideos().getStandardResolution(); - standardResolution.setHeight((long) standardResolutionData.getHeight()); - standardResolution.setWidth((long) standardResolutionData.getWidth()); - standardResolution.setUrl(standardResolutionData.getUrl()); - } - - actObj.setImage(standardResolution); - - return actObj; + return actor; + } + + /** + * Builds the actor. + * @param item MediaFeedData item + * @return a valid ActivityObject + */ + public static ActivityObject buildActor(MediaFeedData item) { + ActivityObject actor = new ActivityObject(); + + try { + Image image = new Image(); + image.setUrl(item.getUser().getProfilePictureUrl()); + + Map<String, Object> extensions = new HashMap<>(); + extensions.put("screenName", item.getUser().getUserName()); + + actor.setDisplayName(item.getUser().getFullName()); + actor.setSummary(item.getUser().getBio()); + actor.setUrl(item.getUser().getWebsiteUrl()); + + actor.setId(formatId(String.valueOf(item.getUser().getId()))); + actor.setImage(image); + actor.setAdditionalProperty("extensions", extensions); + actor.setAdditionalProperty("handle", item.getUser().getUserName()); + } catch (Exception ex) { + LOGGER.error("Exception trying to build actor object: {}", ex.getMessage()); } - /** - * Builds all of the attachments associated with a MediaFeedData object - * - * @param item - * @return - */ - public static List<ActivityObject> buildActivityObjectAttachments(MediaFeedData item) { - List<ActivityObject> attachments = new ArrayList<>(); - - addImageObjects(attachments, item); - addVideoObjects(attachments, item); - - return attachments; + return actor; + } + + /** + * Builds the object. + * @param item the item + * @return a valid Activity Object + */ + public static ActivityObject buildActivityObject(MediaFeedData item) { + ActivityObject actObj = new ActivityObject(); + + actObj.setObjectType(item.getType()); + actObj.setAttachments(buildActivityObjectAttachments(item)); + + Image standardResolution = new Image(); + if (item.getType().equals("image") && item.getImages() != null) { + ImageData standardResolutionData = item.getImages().getStandardResolution(); + standardResolution.setHeight((long) standardResolutionData.getImageHeight()); + standardResolution.setWidth((long) standardResolutionData.getImageWidth()); + standardResolution.setUrl(standardResolutionData.getImageUrl()); + } else if (item.getType().equals("video") && item.getVideos() != null) { + VideoData standardResolutionData = item.getVideos().getStandardResolution(); + standardResolution.setHeight((long) standardResolutionData.getHeight()); + standardResolution.setWidth((long) standardResolutionData.getWidth()); + standardResolution.setUrl(standardResolutionData.getUrl()); } - /** - * Adds any image objects to the attachment field - * @param attachments - * @param item - */ - public static void addImageObjects(List<ActivityObject> attachments, MediaFeedData item) { - Images images = item.getImages(); - - if(images != null) { - try { - ImageData thumbnail = images.getThumbnail(); - ImageData lowResolution = images.getLowResolution(); - - ActivityObject thumbnailObject = new ActivityObject(); - Image thumbnailImage = new Image(); - thumbnailImage.setUrl(thumbnail.getImageUrl()); - thumbnailImage.setHeight((long) thumbnail.getImageHeight()); - thumbnailImage.setWidth((long) thumbnail.getImageWidth()); - thumbnailObject.setImage(thumbnailImage); - thumbnailObject.setObjectType("image"); - - ActivityObject lowResolutionObject = new ActivityObject(); - Image lowResolutionImage = new Image(); - lowResolutionImage.setUrl(lowResolution.getImageUrl()); - lowResolutionImage.setHeight((long) lowResolution.getImageHeight()); - lowResolutionImage.setWidth((long) lowResolution.getImageWidth()); - lowResolutionObject.setImage(lowResolutionImage); - lowResolutionObject.setObjectType("image"); - - attachments.add(thumbnailObject); - attachments.add(lowResolutionObject); - } catch (Exception e) { - LOGGER.error("Failed to add image objects: {}", e.getMessage()); - } - } + actObj.setImage(standardResolution); + + return actObj; + } + + /** + * Builds all of the attachments associated with a MediaFeedData object. + * + * @param item item + * @return result + */ + public static List<ActivityObject> buildActivityObjectAttachments(MediaFeedData item) { + List<ActivityObject> attachments = new ArrayList<>(); + + addImageObjects(attachments, item); + addVideoObjects(attachments, item); + + return attachments; + } + + /** + * Adds any image objects to the attachment field. + * @param attachments attachments + * @param item item + */ + public static void addImageObjects(List<ActivityObject> attachments, MediaFeedData item) { + Images images = item.getImages(); + + if (images != null) { + try { + ImageData thumbnail = images.getThumbnail(); + ImageData lowResolution = images.getLowResolution(); + + ActivityObject thumbnailObject = new ActivityObject(); + Image thumbnailImage = new Image(); + thumbnailImage.setUrl(thumbnail.getImageUrl()); + thumbnailImage.setHeight((long) thumbnail.getImageHeight()); + thumbnailImage.setWidth((long) thumbnail.getImageWidth()); + thumbnailObject.setImage(thumbnailImage); + thumbnailObject.setObjectType("image"); + + ActivityObject lowResolutionObject = new ActivityObject(); + Image lowResolutionImage = new Image(); + lowResolutionImage.setUrl(lowResolution.getImageUrl()); + lowResolutionImage.setHeight((long) lowResolution.getImageHeight()); + lowResolutionImage.setWidth((long) lowResolution.getImageWidth()); + lowResolutionObject.setImage(lowResolutionImage); + lowResolutionObject.setObjectType("image"); + + attachments.add(thumbnailObject); + attachments.add(lowResolutionObject); + } catch (Exception ex) { + LOGGER.error("Failed to add image objects: {}", ex.getMessage()); + } } - - /** - * Adds any video objects to the attachment field - * @param attachments - * @param item - */ - public static void addVideoObjects(List<ActivityObject> attachments, MediaFeedData item) { - Videos videos = item.getVideos(); - - if(videos != null) { - try { - VideoData lowResolutionVideo = videos.getLowResolution(); - - ActivityObject lowResolutionVideoObject = new ActivityObject(); - Image lowResolutionVideoImage = new Image(); - lowResolutionVideoImage.setUrl(lowResolutionVideo.getUrl()); - lowResolutionVideoImage.setHeight((long) lowResolutionVideo.getHeight()); - lowResolutionVideoImage.setWidth((long) lowResolutionVideo.getWidth()); - lowResolutionVideoObject.setImage(lowResolutionVideoImage); - lowResolutionVideoObject.setObjectType("video"); - - attachments.add(lowResolutionVideoObject); - } catch (Exception e) { - LOGGER.error("Failed to add video objects: {}", e.getMessage()); - } - } + } + + /** + * Adds any video objects to the attachment field. + * @param attachments attachments + * @param item item + */ + public static void addVideoObjects(List<ActivityObject> attachments, MediaFeedData item) { + Videos videos = item.getVideos(); + + if (videos != null) { + try { + VideoData lowResolutionVideo = videos.getLowResolution(); + + ActivityObject lowResolutionVideoObject = new ActivityObject(); + Image lowResolutionVideoImage = new Image(); + lowResolutionVideoImage.setUrl(lowResolutionVideo.getUrl()); + lowResolutionVideoImage.setHeight((long) lowResolutionVideo.getHeight()); + lowResolutionVideoImage.setWidth((long) lowResolutionVideo.getWidth()); + lowResolutionVideoObject.setImage(lowResolutionVideoImage); + lowResolutionVideoObject.setObjectType("video"); + + attachments.add(lowResolutionVideoObject); + } catch (Exception ex) { + LOGGER.error("Failed to add video objects: {}", ex.getMessage()); + } } - - /** - * Gets the links from the Instagram event - * @param item the object to use as the source - * @return a list of links corresponding to the expanded URL - */ - public static List<String> getLinks(MediaFeedData item) { - return new ArrayList<>(); + } + + /** + * Gets the links from the Instagram event. + * @param item the object to use as the source + * @return a list of links corresponding to the expanded URL + */ + public static List<String> getLinks(MediaFeedData item) { + return new ArrayList<>(); + } + + /** + * Adds the location extension and populates with the instagram data. + * @param activity the Activity object to update + * @param item the object to use as the source + */ + public static void addLocationExtension(Activity activity, MediaFeedData item) { + Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity); + + if (item.getLocation() != null) { + Map<String, Object> coordinates = new HashMap<>(); + coordinates.put("type", "Point"); + coordinates.put("coordinates", "[" + item.getLocation().getLongitude() + "," + item.getLocation().getLatitude() + "]"); + + extensions.put("coordinates", coordinates); } - - /** - * Adds the location extension and populates with teh instagram data - * @param activity the Activity object to update - * @param item the object to use as the source - */ - public static void addLocationExtension(Activity activity, MediaFeedData item) { - Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity); - - if(item.getLocation() != null) { - Map<String, Object> coordinates = new HashMap<>(); - coordinates.put("type", "Point"); - coordinates.put("coordinates", "[" + item.getLocation().getLongitude() + "," + item.getLocation().getLatitude() + "]"); - - extensions.put("coordinates", coordinates); - } + } + + /** + * Gets the common instagram {@link org.apache.streams.pojo.json.Provider} object. + * @return a provider object representing Instagram + */ + public static Provider getProvider() { + Provider provider = new Provider(); + provider.setId("id:providers:instagram"); + provider.setDisplayName("Instagram"); + return provider; + } + + /** + * Formats the ID to conform with the Apache Streams activity ID convention + * @param idparts the parts of the ID to join + * @return a valid Activity ID in format "id:instagram:part1:part2:...partN" + */ + public static String formatId(String... idparts) { + return Joiner.on(":").join(Lists.asList("id:instagram", idparts)); + } + + /** + * Takes various parameters from the instagram object that are currently not part of the + * activity schema and stores them in a generic extensions attribute. + * @param activity Activity activity + * @param item MediaFeedData item + */ + public static void addInstagramExtensions(Activity activity, MediaFeedData item) { + Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity); + + addLocationExtension(activity, item); + + if (item.getLikes() != null) { + Map<String, Object> likes = new HashMap<>(); + likes.put("count", item.getLikes().getCount()); + extensions.put("likes", likes); } - /** - * Gets the common instagram {@link org.apache.streams.pojo.json.Provider} object - * @return a provider object representing Instagram - */ - public static Provider getProvider() { - Provider provider = new Provider(); - provider.setId("id:providers:instagram"); - provider.setDisplayName("Instagram"); - return provider; - } + extensions.put("hashtags", item.getTags()); - /** - * Formats the ID to conform with the Apache Streams activity ID convention - * @param idparts the parts of the ID to join - * @return a valid Activity ID in format "id:instagram:part1:part2:...partN" - */ - public static String formatId(String... idparts) { - return Joiner.on(":").join(Lists.asList("id:instagram", idparts)); - } + Comments comments = item.getComments(); + String commentsConcat = ""; - /** - * Takes various parameters from the instagram object that are currently not part of teh - * activity schema and stores them in a generic extensions attribute - * @param activity - * @param item - */ - public static void addInstagramExtensions(Activity activity, MediaFeedData item) { - Map<String, Object> extensions = ExtensionUtil.getInstance().ensureExtensions(activity); - - addLocationExtension(activity, item); - - if(item.getLikes() != null) { - Map<String, Object> likes = new HashMap<>(); - likes.put("count", item.getLikes().getCount()); - extensions.put("likes", likes); - } - - extensions.put("hashtags", item.getTags()); - - Comments comments = item.getComments(); - String commentsConcat = ""; - - if(comments != null) { - for (CommentData commentData : comments.getComments()) { - commentsConcat += " " + commentData.getText(); - } - } - if(item.getCaption() != null) { - commentsConcat += " " + item.getCaption().getText(); - } - - extensions.put("keywords", commentsConcat); + if (comments != null) { + for (CommentData commentData : comments.getComments()) { + commentsConcat += " " + commentData.getText(); + } } + if (item.getCaption() != null) { + commentsConcat += " " + item.getCaption().getText(); + } + + extensions.put("keywords", commentsConcat); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramMediaFeedDataConverterIT.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramMediaFeedDataConverterIT.java b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramMediaFeedDataConverterIT.java index af99c14..664cd85 100644 --- a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramMediaFeedDataConverterIT.java +++ b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramMediaFeedDataConverterIT.java @@ -18,86 +18,90 @@ package org.apache.streams.instagram.test.data; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintStream; -import org.apache.commons.lang.StringUtils; import org.apache.streams.data.ActivityConverter; import org.apache.streams.instagram.serializer.InstagramMediaFeedDataConverter; import org.apache.streams.jackson.StreamsJacksonMapper; import org.apache.streams.pojo.json.Activity; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import org.apache.commons.lang.StringUtils; import org.jinstagram.entity.users.feed.MediaFeedData; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintStream; + import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.nullValue; import static org.junit.Assert.assertThat; /** - * Tests conversion of instagram inputs to Activity + * Tests conversion of instagram inputs to Activity. */ + public class InstagramMediaFeedDataConverterIT { - private final static Logger LOGGER = LoggerFactory.getLogger(InstagramMediaFeedDataConverterIT.class); + private static final Logger LOGGER = LoggerFactory.getLogger(InstagramMediaFeedDataConverterIT.class); - // use gson because jInstagram's pojos do - private Gson gson = new Gson(); + // use gson because jInstagram's pojos do + private Gson gson = new Gson(); - // use jackson to write to file output - private ObjectMapper mapper = StreamsJacksonMapper.getInstance(); + // use jackson to write to file output + private ObjectMapper mapper = StreamsJacksonMapper.getInstance(); - @Test - public void InstagramMediaFeedDataConverterITCase() throws Exception { - InputStream is = InstagramMediaFeedDataConverterIT.class.getResourceAsStream("/testMediaFeedObjects.txt"); - InputStreamReader isr = new InputStreamReader(is); - BufferedReader br = new BufferedReader(isr); + @Test + public void InstagramMediaFeedDataConverterITCase() throws Exception { + InputStream is = InstagramMediaFeedDataConverterIT.class.getResourceAsStream("/testMediaFeedObjects.txt"); + InputStreamReader isr = new InputStreamReader(is); + BufferedReader br = new BufferedReader(isr); - PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream("target/test-classes/InstagramMediaFeedDataConverterITCase.txt"))); + PrintStream outStream = new PrintStream( + new BufferedOutputStream( + new FileOutputStream("target/test-classes/InstagramMediaFeedDataConverterITCase.txt"))); - try { - while (br.ready()) { - String line = br.readLine(); - if(!StringUtils.isEmpty(line)) - { - LOGGER.info("raw: {}", line); + try { + while (br.ready()) { + String line = br.readLine(); + if (!StringUtils.isEmpty(line)) { + LOGGER.info("raw: {}", line); - MediaFeedData mediaFeedData = gson.fromJson(line, MediaFeedData.class); + MediaFeedData mediaFeedData = gson.fromJson(line, MediaFeedData.class); - ActivityConverter<MediaFeedData> converter = new InstagramMediaFeedDataConverter(); + ActivityConverter<MediaFeedData> converter = new InstagramMediaFeedDataConverter(); - Activity activity = converter.toActivityList(mediaFeedData).get(0); + Activity activity = converter.toActivityList(mediaFeedData).get(0); - LOGGER.info("activity: {}", activity.toString()); + LOGGER.info("activity: {}", activity.toString()); - assertThat(activity, is(not(nullValue()))); + assertThat(activity, is(not(nullValue()))); - assertThat(activity.getId(), is(not(nullValue()))); - assertThat(activity.getActor(), is(not(nullValue()))); - assertThat(activity.getActor().getId(), is(not(nullValue()))); - assertThat(activity.getVerb(), is(not(nullValue()))); - assertThat(activity.getProvider(), is(not(nullValue()))); + assertThat(activity.getId(), is(not(nullValue()))); + assertThat(activity.getActor(), is(not(nullValue()))); + assertThat(activity.getActor().getId(), is(not(nullValue()))); + assertThat(activity.getVerb(), is(not(nullValue()))); + assertThat(activity.getProvider(), is(not(nullValue()))); - outStream.println(mapper.writeValueAsString(activity)); + outStream.println(mapper.writeValueAsString(activity)); - } + } - } - outStream.flush(); + } + outStream.flush(); - } catch( Exception e ) { - LOGGER.error("Exception: ", e); - outStream.flush(); - Assert.fail(); - } + } catch ( Exception ex ) { + LOGGER.error("Exception: ", ex); + outStream.flush(); + Assert.fail(); } + } } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramUserInfoDataConverterIT.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramUserInfoDataConverterIT.java b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramUserInfoDataConverterIT.java index dcc5eb1..61035d8 100644 --- a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramUserInfoDataConverterIT.java +++ b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/data/InstagramUserInfoDataConverterIT.java @@ -18,18 +18,15 @@ package org.apache.streams.instagram.test.data; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; -import org.apache.commons.lang.StringUtils; -import org.apache.streams.data.ActivityConverter; import org.apache.streams.data.ActivityObjectConverter; -import org.apache.streams.instagram.serializer.InstagramMediaFeedDataConverter; import org.apache.streams.instagram.serializer.InstagramUserInfoDataConverter; import org.apache.streams.jackson.StreamsJacksonMapper; -import org.apache.streams.pojo.json.Activity; import org.apache.streams.pojo.json.ActivityObject; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import org.apache.commons.lang.StringUtils; import org.jinstagram.entity.users.basicinfo.UserInfoData; -import org.jinstagram.entity.users.feed.MediaFeedData; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -49,71 +46,73 @@ import static org.hamcrest.CoreMatchers.nullValue; import static org.junit.Assert.assertThat; /** - * Tests conversion of instagram inputs to Activity + * Tests conversion of instagram inputs to Activity. */ public class InstagramUserInfoDataConverterIT { - private final static Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoDataConverterIT.class); + private static final Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoDataConverterIT.class); - // use gson because jInstagram's pojos do - private Gson gson = new Gson(); + // use gson because jInstagram's pojos do + private Gson gson = new Gson(); - // use jackson to write to file output - private ObjectMapper mapper = StreamsJacksonMapper.getInstance(); + // use jackson to write to file output + private ObjectMapper mapper = StreamsJacksonMapper.getInstance(); - @Test - public void InstagramUserInfoDataConverterIT() throws Exception { - InputStream is = InstagramUserInfoDataConverterIT.class.getResourceAsStream("/testUserInfoData.txt"); - InputStreamReader isr = new InputStreamReader(is); - BufferedReader br = new BufferedReader(isr); + @Test + public void InstagramUserInfoDataConverterIT() throws Exception { + InputStream is = InstagramUserInfoDataConverterIT.class.getResourceAsStream("/testUserInfoData.txt"); + InputStreamReader isr = new InputStreamReader(is); + BufferedReader br = new BufferedReader(isr); - PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream("target/test-classes/InstagramUserInfoDataConverterIT.txt"))); + PrintStream outStream = new PrintStream( + new BufferedOutputStream( + new FileOutputStream("target/test-classes/InstagramUserInfoDataConverterIT.txt"))); - try { - while (br.ready()) { - String line = br.readLine(); - if(!StringUtils.isEmpty(line)) - { - LOGGER.info("raw: {}", line); + try { + while (br.ready()) { + String line = br.readLine(); + if (!StringUtils.isEmpty(line)) { - UserInfoData userInfoData = gson.fromJson(line, UserInfoData.class); + LOGGER.info("raw: {}", line); - ActivityObjectConverter<UserInfoData> converter = new InstagramUserInfoDataConverter(); + UserInfoData userInfoData = gson.fromJson(line, UserInfoData.class); - ActivityObject activityObject = converter.toActivityObject(userInfoData); + ActivityObjectConverter<UserInfoData> converter = new InstagramUserInfoDataConverter(); - LOGGER.info("activityObject: {}", activityObject.toString()); + ActivityObject activityObject = converter.toActivityObject(userInfoData); - assertThat(activityObject, is(not(nullValue()))); + LOGGER.info("activityObject: {}", activityObject.toString()); - assertThat(activityObject.getId(), is(not(nullValue()))); - assertThat(activityObject.getImage(), is(not(nullValue()))); - assertThat(activityObject.getDisplayName(), is(not(nullValue()))); - assertThat(activityObject.getSummary(), is(not(nullValue()))); + assertThat(activityObject, is(not(nullValue()))); - Map<String, Object> extensions = (Map<String, Object>)activityObject.getAdditionalProperties().get("extensions"); - assertThat(extensions, is(not(nullValue()))); - assertThat(extensions.get("following"), is(not(nullValue()))); - assertThat(extensions.get("followers"), is(not(nullValue()))); - assertThat(extensions.get("screenName"), is(not(nullValue()))); - assertThat(extensions.get("posts"), is(not(nullValue()))); + assertThat(activityObject.getId(), is(not(nullValue()))); + assertThat(activityObject.getImage(), is(not(nullValue()))); + assertThat(activityObject.getDisplayName(), is(not(nullValue()))); + assertThat(activityObject.getSummary(), is(not(nullValue()))); - assertThat(activityObject.getAdditionalProperties().get("handle"), is(not(nullValue()))); - assertThat(activityObject.getId(), is(not(nullValue()))); - assertThat(activityObject.getUrl(), is(not(nullValue()))); + Map<String, Object> extensions = (Map<String, Object>)activityObject.getAdditionalProperties().get("extensions"); + assertThat(extensions, is(not(nullValue()))); + assertThat(extensions.get("following"), is(not(nullValue()))); + assertThat(extensions.get("followers"), is(not(nullValue()))); + assertThat(extensions.get("screenName"), is(not(nullValue()))); + assertThat(extensions.get("posts"), is(not(nullValue()))); - assertThat(activityObject.getAdditionalProperties().get("provider"), is(not(nullValue()))); + assertThat(activityObject.getAdditionalProperties().get("handle"), is(not(nullValue()))); + assertThat(activityObject.getId(), is(not(nullValue()))); + assertThat(activityObject.getUrl(), is(not(nullValue()))); - outStream.println(mapper.writeValueAsString(activityObject)); + assertThat(activityObject.getAdditionalProperties().get("provider"), is(not(nullValue()))); - } - } - outStream.flush(); + outStream.println(mapper.writeValueAsString(activityObject)); - } catch( Exception e ) { - LOGGER.error("Exception: ", e); - outStream.flush(); - Assert.fail(); } + } + outStream.flush(); + + } catch ( Exception ex ) { + LOGGER.error("Exception: ", ex); + outStream.flush(); + Assert.fail(); } + } } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramRecentMediaProviderIT.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramRecentMediaProviderIT.java b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramRecentMediaProviderIT.java index ac55e2f..c5a4f77 100644 --- a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramRecentMediaProviderIT.java +++ b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramRecentMediaProviderIT.java @@ -19,6 +19,7 @@ package org.apache.streams.instagram.test.providers; import org.apache.streams.instagram.provider.recentmedia.InstagramRecentMediaProvider; + import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,43 +29,43 @@ import java.io.FileReader; import java.io.LineNumberReader; /** - * Created by sblackmon on 10/12/16. + * Integration Test for InstagramRecentMediaProvider. */ public class InstagramRecentMediaProviderIT { - private static final Logger LOGGER = LoggerFactory.getLogger(InstagramRecentMediaProviderIT.class); - - @Test - public void testInstagramRecentMediaProvider() throws Exception { + private static final Logger LOGGER = LoggerFactory.getLogger(InstagramRecentMediaProviderIT.class); - String configfile = "./target/test-classes/InstagramRecentMediaProviderIT.conf"; - String outfile = "./target/test-classes/InstagramRecentMediaProviderIT.stdout.txt"; + @Test + public void testInstagramRecentMediaProvider() throws Exception { - String[] args = new String[2]; - args[0] = configfile; - args[1] = outfile; + String configfile = "./target/test-classes/InstagramRecentMediaProviderIT.conf"; + String outfile = "./target/test-classes/InstagramRecentMediaProviderIT.stdout.txt"; - Thread testThread = new Thread((Runnable) () -> { - try { - InstagramRecentMediaProvider.main(args); - } catch( Exception e ) { - LOGGER.error("Test Exception!", e); - } - }); - testThread.start(); - testThread.join(60000); + String[] args = new String[2]; + args[0] = configfile; + args[1] = outfile; - File out = new File(outfile); - assert (out.exists()); - assert (out.canRead()); - assert (out.isFile()); + Thread testThread = new Thread((Runnable) () -> { + try { + InstagramRecentMediaProvider.main(args); + } catch ( Exception ex ) { + LOGGER.error("Test Exception!", ex); + } + }); + testThread.start(); + testThread.join(60000); - FileReader outReader = new FileReader(out); - LineNumberReader outCounter = new LineNumberReader(outReader); + File out = new File(outfile); + assert (out.exists()); + assert (out.canRead()); + assert (out.isFile()); - while (outCounter.readLine() != null) { - } + FileReader outReader = new FileReader(out); + LineNumberReader outCounter = new LineNumberReader(outReader); - assert (outCounter.getLineNumber() >= 1); + while (outCounter.readLine() != null) { } + + assert (outCounter.getLineNumber() >= 1); + } } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramUserInfoProviderIT.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramUserInfoProviderIT.java b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramUserInfoProviderIT.java index ec7cd0b..866b254 100644 --- a/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramUserInfoProviderIT.java +++ b/streams-contrib/streams-provider-instagram/src/test/java/org/apache/streams/instagram/test/providers/InstagramUserInfoProviderIT.java @@ -19,6 +19,7 @@ package org.apache.streams.instagram.test.providers; import org.apache.streams.instagram.provider.userinfo.InstagramUserInfoProvider; + import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,39 +33,39 @@ import java.io.LineNumberReader; */ public class InstagramUserInfoProviderIT { - private static final Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoProviderIT.class); + private static final Logger LOGGER = LoggerFactory.getLogger(InstagramUserInfoProviderIT.class); - @Test - public void testInstagramUserInfoProvider() throws Exception { + @Test + public void testInstagramUserInfoProvider() throws Exception { - String configfile = "./target/test-classes/InstagramUserInfoProviderIT.conf"; - String outfile = "./target/test-classes/InstagramUserInfoProviderIT.stdout.txt"; + String configfile = "./target/test-classes/InstagramUserInfoProviderIT.conf"; + String outfile = "./target/test-classes/InstagramUserInfoProviderIT.stdout.txt"; - String[] args = new String[2]; - args[0] = configfile; - args[1] = outfile; + String[] args = new String[2]; + args[0] = configfile; + args[1] = outfile; - Thread testThread = new Thread((Runnable) () -> { - try { - InstagramUserInfoProvider.main(args); - } catch( Exception e ) { - LOGGER.error("Test Exception!", e); - } - }); - testThread.start(); - testThread.join(60000); + Thread testThread = new Thread((Runnable) () -> { + try { + InstagramUserInfoProvider.main(args); + } catch ( Exception ex ) { + LOGGER.error("Test Exception!", ex); + } + }); + testThread.start(); + testThread.join(60000); - File out = new File(outfile); - assert (out.exists()); - assert (out.canRead()); - assert (out.isFile()); + File out = new File(outfile); + assert (out.exists()); + assert (out.canRead()); + assert (out.isFile()); - FileReader outReader = new FileReader(out); - LineNumberReader outCounter = new LineNumberReader(outReader); + FileReader outReader = new FileReader(out); + LineNumberReader outCounter = new LineNumberReader(outReader); - while(outCounter.readLine() != null) {} + while (outCounter.readLine() != null) {} - assert (outCounter.getLineNumber() >= 1); + assert (outCounter.getLineNumber() >= 1); - } + } } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverClient.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverClient.java b/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverClient.java index 05e6120..d9774ed 100644 --- a/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverClient.java +++ b/streams-contrib/streams-provider-moreover/src/main/java/org/apache/streams/moreover/MoreoverClient.java @@ -33,78 +33,89 @@ import java.nio.charset.Charset; import java.util.Date; /** - * + * MoreoverClient is a Client for Moreover. */ public class MoreoverClient { - private static final Logger logger = LoggerFactory.getLogger(MoreoverClient.class); + private static final Logger logger = LoggerFactory.getLogger(MoreoverClient.class); - private static final String BASE_URL = "http://metabase.moreover.com/api/v10/articles?key=%s&limit=%s&sequence_id=%s";; - private final String id; - private String apiKey; - private BigInteger lastSequenceId = BigInteger.ZERO; - //testing purpose only - public long pullTime; - private boolean debug; + private static final String BASE_URL = "http://metabase.moreover.com/api/v10/articles?key=%s&limit=%s&sequence_id=%s";; + private final String id; + private String apiKey; + private BigInteger lastSequenceId = BigInteger.ZERO; + //testing purpose only + public long pullTime; + private boolean debug; - public MoreoverClient(String id, String apiKey, String sequence) { - logger.info("Constructed new client for id:{} key:{} sequence:{}", id, apiKey, sequence); - this.id = id; - this.apiKey = apiKey; - this.lastSequenceId = new BigInteger(sequence); - } + /** + * MoreoverClient constructor. + * @param id id + * @param apiKey apiKey + * @param sequence sequence + */ + public MoreoverClient(String id, String apiKey, String sequence) { + logger.info("Constructed new client for id:{} key:{} sequence:{}", id, apiKey, sequence); + this.id = id; + this.apiKey = apiKey; + this.lastSequenceId = new BigInteger(sequence); + } - public MoreoverResult getArticlesAfter(String sequenceId, int limit) throws IOException { - String urlString = String.format(BASE_URL, this.apiKey, limit, sequenceId); - logger.debug("Making call to {}", urlString); - long start = System.nanoTime(); - MoreoverResult result = new MoreoverResult(id, getArticles(new URL(urlString)), start, System.nanoTime()); - if(!result.getMaxSequencedId().equals(BigInteger.ZERO)) - { - this.lastSequenceId = result.getMaxSequencedId(); - logger.debug("Maximum sequence from last call {}", this.lastSequenceId); - } - else - logger.debug("No maximum sequence returned in last call {}", this.lastSequenceId); - return result; + /** + * get limit ArticlesAfter sequenceId. + * @param sequenceId sequenceId + * @param limit limit + * @return MoreoverResult + * @throws IOException IOException + */ + public MoreoverResult getArticlesAfter(String sequenceId, int limit) throws IOException { + String urlString = String.format(BASE_URL, this.apiKey, limit, sequenceId); + logger.debug("Making call to {}", urlString); + long start = System.nanoTime(); + MoreoverResult result = new MoreoverResult(id, getArticles(new URL(urlString)), start, System.nanoTime()); + if (!result.getMaxSequencedId().equals(BigInteger.ZERO)) { + this.lastSequenceId = result.getMaxSequencedId(); + logger.debug("Maximum sequence from last call {}", this.lastSequenceId); + } else { + logger.debug("No maximum sequence returned in last call {}", this.lastSequenceId); } + return result; + } - public MoreoverResult getNextBatch() throws IOException{ - logger.debug("Getting next results for {} {} {}", this.id, this.apiKey, this.lastSequenceId); - return getArticlesAfter(this.lastSequenceId.toString(), 500); - } + public MoreoverResult getNextBatch() throws IOException { + logger.debug("Getting next results for {} {} {}", this.id, this.apiKey, this.lastSequenceId); + return getArticlesAfter(this.lastSequenceId.toString(), 500); + } - private String getArticles2(URL url) throws IOException { - HttpURLConnection cn = (HttpURLConnection) url.openConnection(); - cn.setRequestMethod("GET"); - cn.addRequestProperty("Content-Type", "text/xml;charset=UTF-8"); - cn.setDoInput(true); - cn.setDoOutput(false); - BufferedReader reader = new BufferedReader(new InputStreamReader(cn.getInputStream(), Charset.forName("UTF-8"))); - String line = null; - StringBuilder builder = new StringBuilder(); - String s = ""; - String result = new String(s.getBytes(Charset.forName("UTF-8")), Charset.forName("UTF-8")); - while((line = reader.readLine()) != null) { - result+=line; - } - pullTime = new Date().getTime(); - return result; + private String getArticles2(URL url) throws IOException { + HttpURLConnection cn = (HttpURLConnection) url.openConnection(); + cn.setRequestMethod("GET"); + cn.addRequestProperty("Content-Type", "text/xml;charset=UTF-8"); + cn.setDoInput(true); + cn.setDoOutput(false); + BufferedReader reader = new BufferedReader(new InputStreamReader(cn.getInputStream(), Charset.forName("UTF-8"))); + String line = null; + StringBuilder builder = new StringBuilder(); + String result = new String("".getBytes(Charset.forName("UTF-8")), Charset.forName("UTF-8")); + while ((line = reader.readLine()) != null) { + result += line; } + pullTime = new Date().getTime(); + return result; + } - private String getArticles(URL url) throws IOException{ - HttpURLConnection cn = (HttpURLConnection) url.openConnection(); - cn.setRequestMethod("GET"); - cn.addRequestProperty("Content-Type", "text/xml;charset=UTF-8"); - cn.setDoInput(true); - cn.setDoOutput(false); - StringWriter writer = new StringWriter(); - IOUtils.copy(new InputStreamReader(cn.getInputStream(), Charset.forName("UTF-8")), writer); - writer.flush(); - pullTime = new Date().getTime(); + private String getArticles(URL url) throws IOException { + HttpURLConnection cn = (HttpURLConnection) url.openConnection(); + cn.setRequestMethod("GET"); + cn.addRequestProperty("Content-Type", "text/xml;charset=UTF-8"); + cn.setDoInput(true); + cn.setDoOutput(false); + StringWriter writer = new StringWriter(); + IOUtils.copy(new InputStreamReader(cn.getInputStream(), Charset.forName("UTF-8")), writer); + writer.flush(); + pullTime = new Date().getTime(); - // added after seeing java.net.SocketException: Too many open files - cn.disconnect(); + // added after seeing java.net.SocketException: Too many open files + cn.disconnect(); - return writer.toString(); - } + return writer.toString(); + } }
