http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java b/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java new file mode 100644 index 0000000..091552f --- /dev/null +++ b/streams-contrib/streams-provider-sysomos/src/test/java/org/apache/streams/sysomos/test/SysomosXmlSerDeIT.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.streams.sysomos.test; + +import com.fasterxml.aalto.stax.InputFactoryImpl; +import com.fasterxml.aalto.stax.OutputFactoryImpl; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule; +import com.fasterxml.jackson.dataformat.xml.XmlFactory; +import com.fasterxml.jackson.dataformat.xml.XmlMapper; +import com.sysomos.xml.BeatApi; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; + +/** + * Tests ability to convert String xml form to {@link com.sysomos.xml.BeatApi} form + */ +public class SysomosXmlSerDeIT { + + private static final Logger LOGGER = LoggerFactory.getLogger(SysomosXmlSerDeIT.class); + + private XmlMapper xmlMapper; + + /** + * before. + */ + @Before + public void before() { + + XmlFactory xmlFactory = new XmlFactory(new InputFactoryImpl(), + new OutputFactoryImpl()); + + JacksonXmlModule module = new JacksonXmlModule(); + + module.setDefaultUseWrapper(false); + + xmlMapper = new XmlMapper(xmlFactory, module); + + xmlMapper + .configure( + DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, + Boolean.TRUE); + xmlMapper + .configure( + DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, + Boolean.TRUE); + xmlMapper + .configure( + DeserializationFeature.USE_JAVA_ARRAY_FOR_JSON_ARRAY, + Boolean.TRUE); + xmlMapper.configure( + DeserializationFeature.READ_ENUMS_USING_TO_STRING, + Boolean.TRUE); + + } + + @Test + public void test() { + + InputStream is = SysomosXmlSerDeIT.class.getResourceAsStream("/sysomos_xmls.txt"); + InputStreamReader isr = new InputStreamReader(is); + BufferedReader br = new BufferedReader(isr); + + try { + while (br.ready()) { + String line = br.readLine(); + LOGGER.debug(line); + + BeatApi ser = xmlMapper.readValue(line, BeatApi.class); + + String des = xmlMapper.writeValueAsString(ser); + LOGGER.debug(des); + } + } catch ( Exception ex ) { + ex.printStackTrace(); + Assert.fail(); + } + } +}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Delete.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Delete.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Delete.json deleted file mode 100644 index 03b801d..0000000 --- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Delete.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "javaType" : "org.apache.streams.twitter.pojo.Delete", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "delete": { - "type": "object", - "javaType" : "org.apache.streams.twitter.pojo.DeleteDetails", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "status": { - "type": "object", - "properties": { - "id": { - "ignore_malformed": false, - "type": "integer" - }, - "user_id": { - "ignore_malformed": false, - "type": "integer" - }, - "id_str": { - "type": "string" - }, - "user_id_str": { - "type": "string" - } - } - } - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Follow.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Follow.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Follow.json deleted file mode 100644 index 320db12..0000000 --- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Follow.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "object", - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "javaType" : "org.apache.streams.twitter.pojo.Follow", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "follower": { - "$ref": "User.json" - }, - "followee": { - "$ref": "User.json" - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json deleted file mode 100644 index ae7a021..0000000 --- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType" : "org.apache.streams.twitter.pojo.FriendList", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "friends": { - "type": "array", - "items": { - "type": "integer" - } - }, - "friends_str": { - "type": "array", - "items": { - "type": "string" - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Retweet.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Retweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Retweet.json deleted file mode 100644 index a449525..0000000 --- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/Retweet.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType" : "org.apache.streams.twitter.pojo.Retweet", - "javaInterfaces": ["java.io.Serializable"], - "extends": {"$ref":"tweet.json"}, - "properties": { - "retweeted_status": { - "type": "object", - "required" : false, - "description" : "Describes the tweet being retweeted.", - "$ref" : "tweet.json" - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterConfiguration.json deleted file mode 100644 index 69048d1..0000000 --- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterConfiguration.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType" : "org.apache.streams.twitter.TwitterConfiguration", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "protocol": { - "type": "string", - "description": "The protocol", - "default": "https" - }, - "host": { - "type": "string", - "description": "The host", - "default": "api.twitter.com" - }, - "port": { - "type": "integer", - "description": "The port", - "default": 443 - }, - "version": { - "type": "string", - "description": "The version", - "default": "1.1" - }, - "endpoint": { - "type": "string", - "description": "The endpoint" - }, - "jsonStoreEnabled": { - "default" : true, - "type": "string" - }, - "oauth": { - "type": "object", - "dynamic": "true", - "javaType" : "org.apache.streams.twitter.TwitterOAuthConfiguration", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "appName": { - "type": "string" - }, - "consumerKey": { - "type": "string" - }, - "consumerSecret": { - "type": "string" - }, - "accessToken": { - "type": "string" - }, - "accessTokenSecret": { - "type": "string" - } - } - }, - "basicauth": { - "type": "object", - "dynamic": "true", - "javaType" : "org.apache.streams.twitter.TwitterBasicAuthConfiguration", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "username": { - "type": "string" - }, - "password": { - "type": "string" - } - } - }, - "retrySleepMs": { - "type": "integer", - "description": "ms to sleep when hitting a rate limit", - "default": 100000 - }, - "retryMax": { - "type": "integer", - "description": "ms to sleep when hitting a rate limit", - "default": 10 - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterFollowingConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterFollowingConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterFollowingConfiguration.json deleted file mode 100644 index 89fc7af..0000000 --- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterFollowingConfiguration.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType" : "org.apache.streams.twitter.TwitterFollowingConfiguration", - "extends": {"$ref":"TwitterUserInformationConfiguration.json"}, - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "ids_only": { - "type": "boolean", - "description": "Whether to collect ids only, or full profiles", - "default": "true" - }, - "max_items": { - "type": "integer", - "description": "Max items per user to collect", - "default": 50000 - }, - "max_pages": { - "type": "integer", - "description": "Max pages per user to request", - "default": 10 - }, - "page_size": { - "type": "integer", - "description": "Max items per page to request", - "default": 5000 - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json deleted file mode 100644 index 6fa2a73..0000000 --- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType" : "org.apache.streams.twitter.TwitterStreamConfiguration", - "extends": {"$ref":"TwitterConfiguration.json"}, - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "includeEntities": { - "type": "string" - }, - "truncated": { - "type": "boolean" - }, - "filter-level": { - "type": "string", - "description": "Setting this parameter to one of none, low, or medium will set the minimum value of the filter_level Tweet attribute required to be included in the stream" - }, - "with": { - "type": "string", - "description": "Typically following or user" - }, - "replies": { - "type": "string", - "description": "Set to all, to see all @replies" - }, - "follow": { - "type": "array", - "description": "A list of user IDs, indicating the users whose Tweets should be delivered on the stream", - "items": { - "type": "integer" - } - }, - "track": { - "type": "array", - "description": "A list of phrases which will be used to determine what Tweets will be delivered on the stream", - "items": { - "type": "string" - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterTimelineProviderConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterTimelineProviderConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterTimelineProviderConfiguration.json deleted file mode 100644 index 37ed60e..0000000 --- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterTimelineProviderConfiguration.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType" : "org.apache.streams.twitter.TwitterTimelineProviderConfiguration", - "extends": {"$ref":"TwitterUserInformationConfiguration.json"}, - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "max_items": { - "type": "integer", - "description": "Max items per user to collect", - "default": 3200 - }, - "max_pages": { - "type": "integer", - "description": "Max items per page to request", - "default": 16 - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterUserInformationConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterUserInformationConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterUserInformationConfiguration.json deleted file mode 100644 index 405c87a..0000000 --- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/TwitterUserInformationConfiguration.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType" : "org.apache.streams.twitter.TwitterUserInformationConfiguration", - "extends": {"$ref":"TwitterConfiguration.json"}, - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "info": { - "type": "array", - "description": "A list of user IDs, indicating the users whose Tweets should be delivered on the stream", - "items": { - "type": "string" - } - }, - "page_size": { - "type": "integer", - "description": "Max items per page to request", - "default": 200 - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/User.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/User.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/User.json deleted file mode 100644 index c6cb798..0000000 --- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/User.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "type": "object", - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "javaType" : "org.apache.streams.twitter.pojo.User", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "location": { - "type": "string" - }, - "default_profile": { - "type": "boolean" - }, - "statuses_count": { - "ignore_malformed": false, - "type": "integer" - }, - "profile_background_tile": { - "type": "boolean" - }, - "lang": { - "type": "string" - }, - "profile_link_color": { - "type": "string" - }, - "id": { - "ignore_malformed": false, - "type": "integer" - }, - "protected": { - "type": "boolean" - }, - "favourites_count": { - "ignore_malformed": false, - "type": "integer" - }, - "profile_text_color": { - "type": "string" - }, - "verified": { - "type": "boolean" - }, - "description": { - "type": "string" - }, - "contributors_enabled": { - "type": "boolean" - }, - "name": { - "type": "string" - }, - "profile_sidebar_border_color": { - "type": "string" - }, - "profile_background_color": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "default_profile_image": { - "type": "boolean" - }, - "followers_count": { - "ignore_malformed": false, - "type": "integer" - }, - "geo_enabled": { - "type": "boolean" - }, - "profile_image_url_https": { - "type": "string" - }, - "profile_background_image_url": { - "type": "string" - }, - "profile_background_image_url_https": { - "type": "string" - }, - "follow_request_sent": { - "type": "boolean" - }, - "url": { - "type": "string" - }, - "utc_offset": { - "ignore_malformed": false, - "type": "integer" - }, - "time_zone": { - "type": "string" - }, - "profile_use_background_image": { - "type": "boolean" - }, - "friends_count": { - "ignore_malformed": false, - "type": "integer" - }, - "profile_sidebar_fill_color": { - "type": "string" - }, - "screen_name": { - "type": "string" - }, - "id_str": { - "type": "string" - }, - "profile_image_url": { - "type": "string" - }, - "is_translator": { - "type": "boolean" - }, - "listed_count": { - "ignore_malformed": false, - "type": "integer" - }, - "status": { - "$ref": "tweet.json" - } - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json deleted file mode 100644 index 23f8365..0000000 --- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "type": "object", - "javaType" : "org.apache.streams.twitter.pojo.UserstreamEvent", - "javaInterfaces": ["java.io.Serializable"], - "description": "", - "properties": { - "created_at": { - "type": "string", - "format" : "date-time" - }, - "event_type": { - "type": "string", - "enum" : [ - "access_revoked", - "block", - "unblock", - "favorite", - "unfavorite", - "follow", - "unfollow", - "list_created", - "list_destroyed", - "list_updated", - "list_member_added", - "list_member_removed", - "list_user_subscribed", - "list_user_unsubscribed", - "user_update" - ] - }, - "source": { - "type": "string", - "items": { - "type": "integer" - } - }, - "target": { - "type": "string", - "items": { - "type": "integer" - } - }, - "target_object": { - "type": "object" - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json deleted file mode 100644 index e529559..0000000 --- a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/tweet.json +++ /dev/null @@ -1,219 +0,0 @@ -{ - "type": "object", - "$schema": "http://json-schema.org/draft-03/schema";, - "$license": [ - "http://www.apache.org/licenses/LICENSE-2.0"; - ], - "id": "#", - "javaType" : "org.apache.streams.twitter.pojo.Tweet", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "text": { - "type": "string" - }, - "retweeted": { - "type": "boolean" - }, - "in_reply_to_screen_name": { - "type": "string" - }, - "truncated": { - "type": "boolean" - }, - "filter_level": { - "type": "string" - }, - "contributors": { - "ignore_malformed": false, - "type": "array", - "items": [{ - "type" : "number" - }] - }, - "place": { - "type": "object", - "javaType" : "org.apache.streams.twitter.pojo.Place", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "id": { - "type": "string" - }, - "bounding_box": { - "type": "object", - "javaType" : "org.apache.streams.twitter.pojo.BoundingBox", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "type": { - "type": "string" - }, - "coordinates": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "number" - } - } - } - } - } - } - } - }, - "entities": { - "type": "object", - "dynamic": "true", - "javaType" : "org.apache.streams.twitter.pojo.Entities", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "user_mentions": { - "type": "array", - "items": { - "type": "object", - "javaType" : "org.apache.streams.twitter.pojo.UserMentions", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "id": { - "ignore_malformed": false, - "type": "integer" - }, - "name": { - "type": "string" - }, - "indices": { - "type": "array", - "items": [{ - "type" : "integer" - }] - }, - "screen_name": { - "type": "string" - }, - "id_str": { - "type": "string" - } - } - } - }, - "hashtags": { - "type": "array", - "items": { - "type": "object", - "javaType": "org.apache.streams.twitter.pojo.Hashtag", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "text": { - "type": "string" - }, - "indices": { - "type": "array", - "items": [{ - "type": "integer" - }] - } - } - } - }, - "urls": { - "type": "array", - "items": { - "type": "object", - "javaType": "org.apache.streams.twitter.Url", - "javaInterfaces": ["java.io.Serializable"], - "properties": { - "expanded_url": { - "type": "string" - }, - "indices": { - "type": "array", - "items": [ - { - "type" : "integer" - } - ] - }, - "display_url": { - "type": "string" - }, - "url": { - "type": "string" - } - } - } - } - } - }, - "in_reply_to_status_id_str": { - "type": "string" - }, - "id": { - "ignore_malformed": false, - "type": "integer" - }, - "in_reply_to_user_id_str": { - "type": "string" - }, - "source": { - "type": "string" - }, - "lang": { - "type": "string" - }, - "favorited": { - "type": "boolean" - }, - "possibly_sensitive": { - "type": "boolean" - }, - "in_reply_to_status_id": { - "ignore_malformed": false, - "type": "integer" - }, - "created_at": { - "type" : "string", - "format" : "date-time" - }, - "in_reply_to_user_id": { - "ignore_malformed": false, - "type": "integer" - }, - "favorite_count": { - "ignore_malformed": false, - "type": "integer" - }, - "retweet_count": { - "ignore_malformed": false, - "type": "integer" - }, - "id_str": { - "type": "string" - }, - "user": { - "$ref": "User.json" - }, - "is_quote_status": { - "type": "boolean" - }, - "quoted_status_id": { - "ignore_malformed": false, - "type": "integer" - }, - "quoted_status": { - "type": "object", - "required" : false, - "description" : "Describes the tweet being quoted.", - "$ref" : "tweet.json" - }, - "retweeted_status_id": { - "type": "integer" - }, - "retweeted_status": { - "type": "object", - "required" : false, - "description" : "Describes the tweet being retweeted.", - "$ref" : "tweet.json" - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterConfiguration.json new file mode 100644 index 0000000..69048d1 --- /dev/null +++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterConfiguration.json @@ -0,0 +1,87 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema";, + "$license": [ + "http://www.apache.org/licenses/LICENSE-2.0"; + ], + "id": "#", + "type": "object", + "javaType" : "org.apache.streams.twitter.TwitterConfiguration", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "protocol": { + "type": "string", + "description": "The protocol", + "default": "https" + }, + "host": { + "type": "string", + "description": "The host", + "default": "api.twitter.com" + }, + "port": { + "type": "integer", + "description": "The port", + "default": 443 + }, + "version": { + "type": "string", + "description": "The version", + "default": "1.1" + }, + "endpoint": { + "type": "string", + "description": "The endpoint" + }, + "jsonStoreEnabled": { + "default" : true, + "type": "string" + }, + "oauth": { + "type": "object", + "dynamic": "true", + "javaType" : "org.apache.streams.twitter.TwitterOAuthConfiguration", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "appName": { + "type": "string" + }, + "consumerKey": { + "type": "string" + }, + "consumerSecret": { + "type": "string" + }, + "accessToken": { + "type": "string" + }, + "accessTokenSecret": { + "type": "string" + } + } + }, + "basicauth": { + "type": "object", + "dynamic": "true", + "javaType" : "org.apache.streams.twitter.TwitterBasicAuthConfiguration", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "username": { + "type": "string" + }, + "password": { + "type": "string" + } + } + }, + "retrySleepMs": { + "type": "integer", + "description": "ms to sleep when hitting a rate limit", + "default": 100000 + }, + "retryMax": { + "type": "integer", + "description": "ms to sleep when hitting a rate limit", + "default": 10 + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterFollowingConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterFollowingConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterFollowingConfiguration.json new file mode 100644 index 0000000..89fc7af --- /dev/null +++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterFollowingConfiguration.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema";, + "$license": [ + "http://www.apache.org/licenses/LICENSE-2.0"; + ], + "id": "#", + "type": "object", + "javaType" : "org.apache.streams.twitter.TwitterFollowingConfiguration", + "extends": {"$ref":"TwitterUserInformationConfiguration.json"}, + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "ids_only": { + "type": "boolean", + "description": "Whether to collect ids only, or full profiles", + "default": "true" + }, + "max_items": { + "type": "integer", + "description": "Max items per user to collect", + "default": 50000 + }, + "max_pages": { + "type": "integer", + "description": "Max pages per user to request", + "default": 10 + }, + "page_size": { + "type": "integer", + "description": "Max items per page to request", + "default": 5000 + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterStreamConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterStreamConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterStreamConfiguration.json new file mode 100644 index 0000000..6fa2a73 --- /dev/null +++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterStreamConfiguration.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema";, + "$license": [ + "http://www.apache.org/licenses/LICENSE-2.0"; + ], + "id": "#", + "type": "object", + "javaType" : "org.apache.streams.twitter.TwitterStreamConfiguration", + "extends": {"$ref":"TwitterConfiguration.json"}, + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "includeEntities": { + "type": "string" + }, + "truncated": { + "type": "boolean" + }, + "filter-level": { + "type": "string", + "description": "Setting this parameter to one of none, low, or medium will set the minimum value of the filter_level Tweet attribute required to be included in the stream" + }, + "with": { + "type": "string", + "description": "Typically following or user" + }, + "replies": { + "type": "string", + "description": "Set to all, to see all @replies" + }, + "follow": { + "type": "array", + "description": "A list of user IDs, indicating the users whose Tweets should be delivered on the stream", + "items": { + "type": "integer" + } + }, + "track": { + "type": "array", + "description": "A list of phrases which will be used to determine what Tweets will be delivered on the stream", + "items": { + "type": "string" + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterTimelineProviderConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterTimelineProviderConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterTimelineProviderConfiguration.json new file mode 100644 index 0000000..37ed60e --- /dev/null +++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterTimelineProviderConfiguration.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema";, + "$license": [ + "http://www.apache.org/licenses/LICENSE-2.0"; + ], + "id": "#", + "type": "object", + "javaType" : "org.apache.streams.twitter.TwitterTimelineProviderConfiguration", + "extends": {"$ref":"TwitterUserInformationConfiguration.json"}, + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "max_items": { + "type": "integer", + "description": "Max items per user to collect", + "default": 3200 + }, + "max_pages": { + "type": "integer", + "description": "Max items per page to request", + "default": 16 + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterUserInformationConfiguration.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterUserInformationConfiguration.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterUserInformationConfiguration.json new file mode 100644 index 0000000..405c87a --- /dev/null +++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/TwitterUserInformationConfiguration.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema";, + "$license": [ + "http://www.apache.org/licenses/LICENSE-2.0"; + ], + "id": "#", + "type": "object", + "javaType" : "org.apache.streams.twitter.TwitterUserInformationConfiguration", + "extends": {"$ref":"TwitterConfiguration.json"}, + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "info": { + "type": "array", + "description": "A list of user IDs, indicating the users whose Tweets should be delivered on the stream", + "items": { + "type": "string" + } + }, + "page_size": { + "type": "integer", + "description": "Max items per page to request", + "default": 200 + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json new file mode 100644 index 0000000..03b801d --- /dev/null +++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Delete.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema";, + "$license": [ + "http://www.apache.org/licenses/LICENSE-2.0"; + ], + "id": "#", + "javaType" : "org.apache.streams.twitter.pojo.Delete", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "delete": { + "type": "object", + "javaType" : "org.apache.streams.twitter.pojo.DeleteDetails", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "status": { + "type": "object", + "properties": { + "id": { + "ignore_malformed": false, + "type": "integer" + }, + "user_id": { + "ignore_malformed": false, + "type": "integer" + }, + "id_str": { + "type": "string" + }, + "user_id_str": { + "type": "string" + } + } + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json new file mode 100644 index 0000000..320db12 --- /dev/null +++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Follow.json @@ -0,0 +1,18 @@ +{ + "type": "object", + "$schema": "http://json-schema.org/draft-03/schema";, + "$license": [ + "http://www.apache.org/licenses/LICENSE-2.0"; + ], + "id": "#", + "javaType" : "org.apache.streams.twitter.pojo.Follow", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "follower": { + "$ref": "User.json" + }, + "followee": { + "$ref": "User.json" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json new file mode 100644 index 0000000..ae7a021 --- /dev/null +++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/FriendList.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema";, + "$license": [ + "http://www.apache.org/licenses/LICENSE-2.0"; + ], + "id": "#", + "type": "object", + "javaType" : "org.apache.streams.twitter.pojo.FriendList", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "friends": { + "type": "array", + "items": { + "type": "integer" + } + }, + "friends_str": { + "type": "array", + "items": { + "type": "string" + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json new file mode 100644 index 0000000..a449525 --- /dev/null +++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/Retweet.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema";, + "$license": [ + "http://www.apache.org/licenses/LICENSE-2.0"; + ], + "id": "#", + "type": "object", + "javaType" : "org.apache.streams.twitter.pojo.Retweet", + "javaInterfaces": ["java.io.Serializable"], + "extends": {"$ref":"tweet.json"}, + "properties": { + "retweeted_status": { + "type": "object", + "required" : false, + "description" : "Describes the tweet being retweeted.", + "$ref" : "tweet.json" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json new file mode 100644 index 0000000..c6cb798 --- /dev/null +++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/User.json @@ -0,0 +1,129 @@ +{ + "type": "object", + "$schema": "http://json-schema.org/draft-03/schema";, + "$license": [ + "http://www.apache.org/licenses/LICENSE-2.0"; + ], + "id": "#", + "javaType" : "org.apache.streams.twitter.pojo.User", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "location": { + "type": "string" + }, + "default_profile": { + "type": "boolean" + }, + "statuses_count": { + "ignore_malformed": false, + "type": "integer" + }, + "profile_background_tile": { + "type": "boolean" + }, + "lang": { + "type": "string" + }, + "profile_link_color": { + "type": "string" + }, + "id": { + "ignore_malformed": false, + "type": "integer" + }, + "protected": { + "type": "boolean" + }, + "favourites_count": { + "ignore_malformed": false, + "type": "integer" + }, + "profile_text_color": { + "type": "string" + }, + "verified": { + "type": "boolean" + }, + "description": { + "type": "string" + }, + "contributors_enabled": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "profile_sidebar_border_color": { + "type": "string" + }, + "profile_background_color": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "default_profile_image": { + "type": "boolean" + }, + "followers_count": { + "ignore_malformed": false, + "type": "integer" + }, + "geo_enabled": { + "type": "boolean" + }, + "profile_image_url_https": { + "type": "string" + }, + "profile_background_image_url": { + "type": "string" + }, + "profile_background_image_url_https": { + "type": "string" + }, + "follow_request_sent": { + "type": "boolean" + }, + "url": { + "type": "string" + }, + "utc_offset": { + "ignore_malformed": false, + "type": "integer" + }, + "time_zone": { + "type": "string" + }, + "profile_use_background_image": { + "type": "boolean" + }, + "friends_count": { + "ignore_malformed": false, + "type": "integer" + }, + "profile_sidebar_fill_color": { + "type": "string" + }, + "screen_name": { + "type": "string" + }, + "id_str": { + "type": "string" + }, + "profile_image_url": { + "type": "string" + }, + "is_translator": { + "type": "boolean" + }, + "listed_count": { + "ignore_malformed": false, + "type": "integer" + }, + "status": { + "$ref": "tweet.json" + } + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json new file mode 100644 index 0000000..23f8365 --- /dev/null +++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/UserstreamEvent.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/draft-03/schema";, + "$license": [ + "http://www.apache.org/licenses/LICENSE-2.0"; + ], + "id": "#", + "type": "object", + "javaType" : "org.apache.streams.twitter.pojo.UserstreamEvent", + "javaInterfaces": ["java.io.Serializable"], + "description": "", + "properties": { + "created_at": { + "type": "string", + "format" : "date-time" + }, + "event_type": { + "type": "string", + "enum" : [ + "access_revoked", + "block", + "unblock", + "favorite", + "unfavorite", + "follow", + "unfollow", + "list_created", + "list_destroyed", + "list_updated", + "list_member_added", + "list_member_removed", + "list_user_subscribed", + "list_user_unsubscribed", + "user_update" + ] + }, + "source": { + "type": "string", + "items": { + "type": "integer" + } + }, + "target": { + "type": "string", + "items": { + "type": "integer" + } + }, + "target_object": { + "type": "object" + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json new file mode 100644 index 0000000..e529559 --- /dev/null +++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/org/apache/streams/twitter/pojo/tweet.json @@ -0,0 +1,219 @@ +{ + "type": "object", + "$schema": "http://json-schema.org/draft-03/schema";, + "$license": [ + "http://www.apache.org/licenses/LICENSE-2.0"; + ], + "id": "#", + "javaType" : "org.apache.streams.twitter.pojo.Tweet", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "text": { + "type": "string" + }, + "retweeted": { + "type": "boolean" + }, + "in_reply_to_screen_name": { + "type": "string" + }, + "truncated": { + "type": "boolean" + }, + "filter_level": { + "type": "string" + }, + "contributors": { + "ignore_malformed": false, + "type": "array", + "items": [{ + "type" : "number" + }] + }, + "place": { + "type": "object", + "javaType" : "org.apache.streams.twitter.pojo.Place", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "id": { + "type": "string" + }, + "bounding_box": { + "type": "object", + "javaType" : "org.apache.streams.twitter.pojo.BoundingBox", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "type": { + "type": "string" + }, + "coordinates": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + } + } + } + }, + "entities": { + "type": "object", + "dynamic": "true", + "javaType" : "org.apache.streams.twitter.pojo.Entities", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "user_mentions": { + "type": "array", + "items": { + "type": "object", + "javaType" : "org.apache.streams.twitter.pojo.UserMentions", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "id": { + "ignore_malformed": false, + "type": "integer" + }, + "name": { + "type": "string" + }, + "indices": { + "type": "array", + "items": [{ + "type" : "integer" + }] + }, + "screen_name": { + "type": "string" + }, + "id_str": { + "type": "string" + } + } + } + }, + "hashtags": { + "type": "array", + "items": { + "type": "object", + "javaType": "org.apache.streams.twitter.pojo.Hashtag", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "text": { + "type": "string" + }, + "indices": { + "type": "array", + "items": [{ + "type": "integer" + }] + } + } + } + }, + "urls": { + "type": "array", + "items": { + "type": "object", + "javaType": "org.apache.streams.twitter.Url", + "javaInterfaces": ["java.io.Serializable"], + "properties": { + "expanded_url": { + "type": "string" + }, + "indices": { + "type": "array", + "items": [ + { + "type" : "integer" + } + ] + }, + "display_url": { + "type": "string" + }, + "url": { + "type": "string" + } + } + } + } + } + }, + "in_reply_to_status_id_str": { + "type": "string" + }, + "id": { + "ignore_malformed": false, + "type": "integer" + }, + "in_reply_to_user_id_str": { + "type": "string" + }, + "source": { + "type": "string" + }, + "lang": { + "type": "string" + }, + "favorited": { + "type": "boolean" + }, + "possibly_sensitive": { + "type": "boolean" + }, + "in_reply_to_status_id": { + "ignore_malformed": false, + "type": "integer" + }, + "created_at": { + "type" : "string", + "format" : "date-time" + }, + "in_reply_to_user_id": { + "ignore_malformed": false, + "type": "integer" + }, + "favorite_count": { + "ignore_malformed": false, + "type": "integer" + }, + "retweet_count": { + "ignore_malformed": false, + "type": "integer" + }, + "id_str": { + "type": "string" + }, + "user": { + "$ref": "User.json" + }, + "is_quote_status": { + "type": "boolean" + }, + "quoted_status_id": { + "ignore_malformed": false, + "type": "integer" + }, + "quoted_status": { + "type": "object", + "required" : false, + "description" : "Describes the tweet being quoted.", + "$ref" : "tweet.json" + }, + "retweeted_status_id": { + "type": "integer" + }, + "retweeted_status": { + "type": "object", + "required" : false, + "description" : "Describes the tweet being retweeted.", + "$ref" : "tweet.json" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/processor/YoutubeTypeConverter.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/processor/YoutubeTypeConverter.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/processor/YoutubeTypeConverter.java deleted file mode 100644 index d8967d1..0000000 --- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/processor/YoutubeTypeConverter.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.youtube.processor; - -import org.apache.streams.core.StreamsDatum; -import org.apache.streams.core.StreamsProcessor; -import org.apache.streams.jackson.StreamsJacksonMapper; -import org.apache.streams.pojo.json.Activity; - -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.google.api.services.youtube.model.Channel; -import com.google.api.services.youtube.model.Video; -import com.youtube.serializer.YoutubeActivityUtil; -import com.youtube.serializer.YoutubeChannelDeserializer; -import com.youtube.serializer.YoutubeEventClassifier; -import com.youtube.serializer.YoutubeVideoDeserializer; -import org.apache.commons.lang.NotImplementedException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Queue; - -public class YoutubeTypeConverter implements StreamsProcessor { - - public static final String STREAMS_ID = "YoutubeTypeConverter"; - - private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeTypeConverter.class); - - private StreamsJacksonMapper mapper; - private Queue<Video> inQueue; - private Queue<StreamsDatum> outQueue; - private int count = 0; - - public YoutubeTypeConverter() {} - - @Override - public String getId() { - return STREAMS_ID; - } - - @Override - public List<StreamsDatum> process(StreamsDatum streamsDatum) { - StreamsDatum result = null; - - try { - Object item = streamsDatum.getDocument(); - - LOGGER.debug("{} processing {}", STREAMS_ID, item.getClass()); - Activity activity; - - if (item instanceof String) { - item = deserializeItem(item); - } - - if (item instanceof Video) { - activity = new Activity(); - YoutubeActivityUtil.updateActivity((Video)item, activity, streamsDatum.getId()); - } else if (item instanceof Channel) { - activity = new Activity(); - YoutubeActivityUtil.updateActivity((Channel)item, activity, null); - } else { - throw new NotImplementedException("Type conversion not implement for type : " + item.getClass().getName()); - } - - if (activity != null) { - result = new StreamsDatum(activity); - count++; - } - } catch (Exception ex) { - LOGGER.error("Exception while converting Video to Activity: {}", ex); - } - - if ( result != null ) { - List<StreamsDatum> streamsDatumList = new ArrayList<>(); - streamsDatumList.add(result); - return streamsDatumList; - } else { - return new ArrayList<>(); - } - } - - private Object deserializeItem(Object item) { - try { - Class klass = YoutubeEventClassifier.detectClass((String) item); - if (klass.equals(Video.class)) { - item = mapper.readValue((String) item, Video.class); - } else if (klass.equals(Channel.class)) { - item = mapper.readValue((String) item, Channel.class); - } - } catch (Exception ex) { - LOGGER.error("Exception while trying to deserializeItem: {}", ex); - } - - return item; - } - - @Override - public void prepare(Object configurationObject) { - mapper = StreamsJacksonMapper.getInstance(); - - SimpleModule simpleModule = new SimpleModule(); - simpleModule.addDeserializer(Video.class, new YoutubeVideoDeserializer()); - mapper.registerModule(simpleModule); - simpleModule = new SimpleModule(); - simpleModule.addDeserializer(Channel.class, new YoutubeChannelDeserializer()); - mapper.registerModule(simpleModule); - } - - @Override - public void cleanUp() {} -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelDataCollector.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelDataCollector.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelDataCollector.java deleted file mode 100644 index fd238db..0000000 --- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelDataCollector.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.youtube.provider; - -import org.apache.streams.core.StreamsDatum; -import org.apache.streams.google.gplus.configuration.UserInfo; -import org.apache.streams.jackson.StreamsJacksonMapper; -import org.apache.streams.util.api.requests.backoff.BackOffStrategy; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.googleapis.json.GoogleJsonResponseException; -import com.google.api.services.youtube.YouTube; -import com.google.api.services.youtube.model.Channel; -import com.google.gson.Gson; -import org.apache.commons.lang3.StringUtils; -import org.apache.youtube.pojo.YoutubeConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.concurrent.BlockingQueue; - -/** - * Collects YoutubeChannelData on behalf of YoutubeChannelProvider. - */ -public class YoutubeChannelDataCollector extends YoutubeDataCollector { - - private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeChannelDataCollector.class); - private static final String CONTENT = "snippet,contentDetails,statistics,topicDetails"; - private static final ObjectMapper MAPPER = StreamsJacksonMapper.getInstance(); - private static final int MAX_ATTEMPTS = 5; - - private YouTube youTube; - private BlockingQueue<StreamsDatum> queue; - private BackOffStrategy strategy; - private UserInfo userInfo; - private YoutubeConfiguration youtubeConfig; - - /** - * YoutubeChannelDataCollector constructor. - * @param youTube YouTube - * @param queue BlockingQueue of StreamsDatum - * @param strategy BackOffStrategy - * @param userInfo UserInfo - * @param youtubeConfig YoutubeConfiguration - */ - public YoutubeChannelDataCollector( - YouTube youTube, - BlockingQueue<StreamsDatum> queue, - BackOffStrategy strategy, - UserInfo userInfo, - YoutubeConfiguration youtubeConfig) { - this.youTube = youTube; - this.queue = queue; - this.strategy = strategy; - this.userInfo = userInfo; - this.youtubeConfig = youtubeConfig; - } - - @Override - public void run() { - Gson gson = new Gson(); - try { - int attempt = 0; - YouTube.Channels.List channelLists = this.youTube.channels().list(CONTENT).setId(this.userInfo.getUserId()).setKey(this.youtubeConfig.getApiKey()); - boolean tryAgain = false; - do { - try { - List<Channel> channels = channelLists.execute().getItems(); - for (Channel channel : channels) { - String json = gson.toJson(channel); - this.queue.put(new StreamsDatum(json, channel.getId())); - } - if (StringUtils.isEmpty(channelLists.getPageToken())) { - channelLists = null; - } else { - channelLists = this.youTube.channels().list(CONTENT).setId(this.userInfo.getUserId()).setOauthToken(this.youtubeConfig.getApiKey()) - .setPageToken(channelLists.getPageToken()); - } - } catch (GoogleJsonResponseException gjre) { - LOGGER.warn("GoogleJsonResposneException caught : {}", gjre); - tryAgain = backoffAndIdentifyIfRetry(gjre, this.strategy); - ++attempt; - } catch (Throwable throwable) { - LOGGER.warn("Unable to get channel info for id : {}", this.userInfo.getUserId()); - LOGGER.warn("Excpection thrown while trying to get channel info : {}", throwable); - } - } - while ((tryAgain && attempt < MAX_ATTEMPTS) || channelLists != null); - - } catch (Throwable throwable) { - LOGGER.warn(throwable.getMessage()); - } - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelProvider.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelProvider.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelProvider.java deleted file mode 100644 index d9b1e14..0000000 --- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeChannelProvider.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.youtube.provider; - -import org.apache.streams.config.ComponentConfigurator; -import org.apache.streams.config.StreamsConfiguration; -import org.apache.streams.config.StreamsConfigurator; -import org.apache.streams.core.StreamsDatum; -import org.apache.streams.google.gplus.configuration.UserInfo; -import org.apache.streams.jackson.StreamsJacksonMapper; -import org.apache.streams.util.api.requests.backoff.BackOffStrategy; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.services.youtube.YouTube; -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.Uninterruptibles; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; -import com.typesafe.config.ConfigParseOptions; -import org.apache.youtube.pojo.YoutubeConfiguration; - -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.PrintStream; -import java.util.Iterator; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.TimeUnit; - -/** - * Retrieve recent activity from a list of channels. - */ -public class YoutubeChannelProvider extends YoutubeProvider { - - public YoutubeChannelProvider() { - super(); - } - - public YoutubeChannelProvider(YoutubeConfiguration config) { - super(config); - } - - @Override - protected Runnable getDataCollector(BackOffStrategy strategy, BlockingQueue<StreamsDatum> queue, YouTube youtube, UserInfo userInfo) { - return new YoutubeChannelDataCollector(youtube, queue, strategy, userInfo, this.config); - } - - /** - * To use from command line: - * - * <p/> - * Supply (at least) the following required configuration in application.conf: - * - * <p/> - * youtube.oauth.pathToP12KeyFile - * youtube.oauth.serviceAccountEmailAddress - * youtube.apiKey - * youtube.youtubeUsers - * - * <p/> - * Launch using: - * - * <p/> - * mvn exec:java -Dexec.mainClass=org.apache.streams.youtube.provider.YoutubeUserActivityProvider -Dexec.args="application.conf tweets.json" - * - * @param args args - * @throws Exception Exception - */ - public static void main(String[] args) throws Exception { - - Preconditions.checkArgument(args.length >= 2); - - String configfile = args[0]; - String outfile = args[1]; - - Config reference = ConfigFactory.load(); - File file = new File(configfile); - assert (file.exists()); - Config testResourceConfig = ConfigFactory.parseFileAnySyntax(file, ConfigParseOptions.defaults().setAllowMissing(false)); - - Config typesafe = testResourceConfig.withFallback(reference).resolve(); - - StreamsConfiguration streamsConfiguration = StreamsConfigurator.detectConfiguration(typesafe); - YoutubeConfiguration config = new ComponentConfigurator<>(YoutubeConfiguration.class).detectConfiguration(typesafe, "youtube"); - YoutubeChannelProvider provider = new YoutubeChannelProvider(config); - - ObjectMapper mapper = StreamsJacksonMapper.getInstance(); - - PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outfile))); - provider.prepare(config); - provider.startStream(); - do { - Uninterruptibles.sleepUninterruptibly(streamsConfiguration.getBatchFrequencyMs(), TimeUnit.MILLISECONDS); - Iterator<StreamsDatum> iterator = provider.readCurrent().iterator(); - while (iterator.hasNext()) { - StreamsDatum datum = iterator.next(); - String json; - try { - if ( datum.getDocument() instanceof String ) { - json = (String) datum.getDocument(); - } else { - json = mapper.writeValueAsString(datum.getDocument()); - } - outStream.println(json); - } catch (JsonProcessingException ex) { - System.err.println(ex.getMessage()); - } - } - } - while ( provider.isRunning()); - provider.cleanUp(); - outStream.flush(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b71cce83/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeDataCollector.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeDataCollector.java b/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeDataCollector.java deleted file mode 100644 index 3eede18..0000000 --- a/streams-contrib/streams-provider-youtube/src/main/java/com/youtube/provider/YoutubeDataCollector.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package com.youtube.provider; - -import org.apache.streams.util.api.requests.backoff.BackOffException; -import org.apache.streams.util.api.requests.backoff.BackOffStrategy; - -import com.google.api.client.googleapis.json.GoogleJsonResponseException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Base Collector for Youtube Data. - */ -public abstract class YoutubeDataCollector implements Runnable { - - private static final Logger LOGGER = LoggerFactory.getLogger(YoutubeDataCollector.class); - - /** - * Looks at the status code of the expception. If the code indicates that the request should be retried, - * it executes the back off strategy and returns true. - * @param gjre - * @param backOff - * @return returns true if the error code of the exception indicates the request should be retried. - */ - public boolean backoffAndIdentifyIfRetry(GoogleJsonResponseException gjre, BackOffStrategy backOff) throws BackOffException { - boolean tryAgain = false; - - switch (gjre.getStatusCode()) { - case 400 : - LOGGER.warn("Bad Request : {}", gjre); - break; - case 401 : - LOGGER.warn("Invalid Credentials : {}", gjre); - break; - case 403 : - LOGGER.warn("Possible rate limit exception. Retrying. : {}", gjre.getMessage()); - backOff.backOff(); - tryAgain = true; - break; - case 503 : - LOGGER.warn("Google Backend Service Error : {}", gjre); - break; - default: - LOGGER.warn("Google Service returned error : {}", gjre); - tryAgain = true; - backOff.backOff(); - break; - } - - return tryAgain; - } -} \ No newline at end of file
