IGNITE-530 Initial implementation
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a39681ef Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a39681ef Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a39681ef Branch: refs/heads/ignite-801 Commit: a39681ef54206ac23f3dff9dab930dc2596d9870 Parents: 480d457 Author: sylentprayer <un4given> Authored: Thu Nov 19 15:32:24 2015 +0300 Committer: Anton Vinogradov <[email protected]> Committed: Thu Nov 19 15:32:24 2015 +0300 ---------------------------------------------------------------------- modules/twitter/README.txt | 32 ++ modules/twitter/licenses/apache-2.0.txt | 202 +++++++++++++ modules/twitter/pom.xml | 116 ++++++++ .../ignite/stream/twitter/OAuthSettings.java | 86 ++++++ .../ignite/stream/twitter/TwitterStreamer.java | 295 +++++++++++++++++++ .../twitter/IgniteTwitterStreamerTest.java | 234 +++++++++++++++ .../twitter/IgniteTwitterStreamerTestSuite.java | 32 ++ .../stream/twitter/TwitterStreamerImpl.java | 79 +++++ pom.xml | 1 + 9 files changed, 1077 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/a39681ef/modules/twitter/README.txt ---------------------------------------------------------------------- diff --git a/modules/twitter/README.txt b/modules/twitter/README.txt new file mode 100644 index 0000000..1c412f9 --- /dev/null +++ b/modules/twitter/README.txt @@ -0,0 +1,32 @@ +Apache Ignite Twitter Streamer Module +------------------------ + +Apache Ignite Twitter Streamer module provides streaming from Twitter to Ignite cache. + +To enable Twitter Streamer module when starting a standalone node, move 'optional/ignite-twitter' folder to +'libs' folder before running 'ignite.{sh|bat}' script. The content of the module folder will +be added to classpath in this case. + +Importing Ignite Twitter Streamer Module In Maven Project +------------------------------------- + +If you are using Maven to manage dependencies of your project, you can add JCL module +dependency like this (replace '${ignite.version}' with actual Ignite version you are +interested in): + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd";> + ... + <dependencies> + ... + <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-twitter</artifactId> + <version>${ignite.version}</version> + </dependency> + ... + </dependencies> + ... +</project> http://git-wip-us.apache.org/repos/asf/ignite/blob/a39681ef/modules/twitter/licenses/apache-2.0.txt ---------------------------------------------------------------------- diff --git a/modules/twitter/licenses/apache-2.0.txt b/modules/twitter/licenses/apache-2.0.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/modules/twitter/licenses/apache-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. http://git-wip-us.apache.org/repos/asf/ignite/blob/a39681ef/modules/twitter/pom.xml ---------------------------------------------------------------------- diff --git a/modules/twitter/pom.xml b/modules/twitter/pom.xml new file mode 100644 index 0000000..21e6f51 --- /dev/null +++ b/modules/twitter/pom.xml @@ -0,0 +1,116 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + POM file. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-parent</artifactId> + <version>1</version> + <relativePath>../../parent</relativePath> + </parent> + + <artifactId>ignite-twitter</artifactId> + <version>1.5.0-SNAPSHOT</version> + <url>http://ignite.apache.org</url> + + <dependencies> + <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-log4j</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-spring</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-core</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>hbc-twitter4j</artifactId> + <version>2.2.0</version> + </dependency> + <dependency> + <groupId>com.github.tomakehurst</groupId> + <artifactId>wiremock</artifactId> + <version>1.57</version> + <classifier>standalone</classifier> + <exclusions> + <exclusion> + <groupId>org.mortbay.jetty</groupId> + <artifactId>jetty</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + </exclusion> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + <exclusion> + <groupId>org.skyscreamer</groupId> + <artifactId>jsonassert</artifactId> + </exclusion> + <exclusion> + <groupId>xmlunit</groupId> + <artifactId>xmlunit</artifactId> + </exclusion> + <exclusion> + <groupId>com.jayway.jsonpath</groupId> + <artifactId>json-path</artifactId> + </exclusion> + <exclusion> + <groupId>net.sf.jopt-simple</groupId> + <artifactId>jopt-simple</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/ignite/blob/a39681ef/modules/twitter/src/main/java/org/apache/ignite/stream/twitter/OAuthSettings.java ---------------------------------------------------------------------- diff --git a/modules/twitter/src/main/java/org/apache/ignite/stream/twitter/OAuthSettings.java b/modules/twitter/src/main/java/org/apache/ignite/stream/twitter/OAuthSettings.java new file mode 100644 index 0000000..c2e46a5 --- /dev/null +++ b/modules/twitter/src/main/java/org/apache/ignite/stream/twitter/OAuthSettings.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.stream.twitter; + +import org.jetbrains.annotations.NotNull; + +/** + * OAuth keys holder. + */ +public class OAuthSettings { + /** */ + private final String consumerKey; + + /** */ + private final String consumerSecret; + + /** */ + private final String accessToken; + + /** */ + private final String accessTokenSecret; + + /** + * @param consumerKey Consumer key. + * @param consumerSecret Consumer secret. + * @param accessToken Access token. + * @param accessTokenSecret Access secret token. + */ + public OAuthSettings( + @NotNull String consumerKey, + @NotNull String consumerSecret, + @NotNull String accessToken, + @NotNull String accessTokenSecret) { + this.consumerKey = consumerKey; + this.consumerSecret = consumerSecret; + this.accessToken = accessToken; + this.accessTokenSecret = accessTokenSecret; + } + + /** + * @return Consumer key. + */ + @NotNull + public String getConsumerKey() { + return consumerKey; + } + + /** + * @return Consumer secret. + */ + @NotNull + public String getConsumerSecret() { + return consumerSecret; + } + + /** + * @return Access token. + */ + @NotNull + public String getAccessToken() { + return accessToken; + } + + /** + * @return Access token secret. + */ + @NotNull + public String getAccessTokenSecret() { + return accessTokenSecret; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/a39681ef/modules/twitter/src/main/java/org/apache/ignite/stream/twitter/TwitterStreamer.java ---------------------------------------------------------------------- diff --git a/modules/twitter/src/main/java/org/apache/ignite/stream/twitter/TwitterStreamer.java b/modules/twitter/src/main/java/org/apache/ignite/stream/twitter/TwitterStreamer.java new file mode 100644 index 0000000..309cc52 --- /dev/null +++ b/modules/twitter/src/main/java/org/apache/ignite/stream/twitter/TwitterStreamer.java @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.stream.twitter; + +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import com.twitter.hbc.ClientBuilder; +import com.twitter.hbc.core.Client; +import com.twitter.hbc.core.HttpConstants; +import com.twitter.hbc.core.HttpHosts; +import com.twitter.hbc.core.endpoint.DefaultStreamingEndpoint; +import com.twitter.hbc.core.endpoint.SitestreamEndpoint; +import com.twitter.hbc.core.endpoint.StatusesFilterEndpoint; +import com.twitter.hbc.core.endpoint.StatusesFirehoseEndpoint; +import com.twitter.hbc.core.endpoint.StatusesSampleEndpoint; +import com.twitter.hbc.core.endpoint.StreamingEndpoint; +import com.twitter.hbc.core.endpoint.UserstreamEndpoint; +import com.twitter.hbc.core.processor.StringDelimitedProcessor; +import com.twitter.hbc.httpclient.auth.Authentication; +import com.twitter.hbc.httpclient.auth.OAuth1; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.util.typedef.internal.A; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.stream.StreamAdapter; + +/** + * Streamer that consumes from a Twitter Streaming API and feeds transformed key-value pairs, + * by default <tweetId, text>, into an {@link IgniteDataStreamer} instance. + * <p> + * This streamer uses https://dev.twitter.com/streaming API and supports Public API, User Streams, + * Site Streams and Firehose. + * <p> + * This Streamer features: + * <ul> + * <li>Supports OAuth1 authentication scheme. + * <br/> BasicAuth not supported by Streaming API https://dev.twitter.com/streaming/overview/connecting</li> + * <li>Provide all params in apiParams map. https://dev.twitter.com/streaming/overview/request-parameters</li> + * </ul> + */ +public class TwitterStreamer<K, V> extends StreamAdapter<String, K, V> { + /** Logger. */ + protected IgniteLogger log; + + /** Threads count used to transform tweets. */ + private int threadsCount = 1; + + /** Twitter Streaming API params. See https://dev.twitter.com/streaming/overview/request-parameters */ + private Map<String, String> apiParams; + + /** Twitter streaming API endpoint example, '/statuses/filter.json' or '/statuses/firehose.json' */ + private String endpointUrl; + + /** OAuth params holder */ + private OAuthSettings oAuthSettings; + + /** shared variable to communicate/signal that streamer is already running or can be started */ + private final AtomicInteger running = new AtomicInteger(); + + /** + * Size of buffer for streaming, as for some tracking terms traffic can be low and for others high, this is + * configurable + */ + private Integer bufferCapacity = 100000; + + /** Twitter streaming client (Twitter HBC) to interact with stream */ + private Client client; + + /** Process stream asynchronously */ + private ExecutorService tweetStreamProcessor; + + /** Param key name constant for Site streaming */ + private final String SITE_USER_ID_KEY = "follow"; + + /** + * @param oAuthSettings OAuth Settings + */ + public TwitterStreamer(OAuthSettings oAuthSettings) { + this.oAuthSettings = oAuthSettings; + } + + /** + * Starts streamer. + */ + public void start() { + if (!running.compareAndSet(0, 1)) + throw new IgniteException("Attempted to start an already started Twitter Streamer"); + + validateConfig(); + + log = getIgnite().log(); + + final BlockingQueue<String> tweetQueue = new LinkedBlockingQueue<>(bufferCapacity); + + client = getClient(tweetQueue); + + client.connect(); + + tweetStreamProcessor = Executors.newFixedThreadPool(threadsCount); + + for (int i = 0; i < threadsCount; i++) { + Callable<Boolean> task = new Callable<Boolean>() { + + @Override + public Boolean call() { + while (true) { + try { + String tweet = tweetQueue.take(); + + addMessage(tweet); + } + catch (InterruptedException e) { + U.warn(log, "Tweets transformation was interrupted", e); + + return true; + } + } + } + }; + + tweetStreamProcessor.submit(task); + } + } + + /** + * Stops streamer. + */ + public void stop() { + if (running.get() == 0) + throw new IgniteException("Attempted to stop an already stopped Twitter Streamer"); + + tweetStreamProcessor.shutdownNow(); + + client.stop(); + + running.compareAndSet(1, 0); + } + + /** + * Validates config at start. + */ + protected void validateConfig() { + A.notNull(getStreamer(), "Streamer"); + A.notNull(getIgnite(), "Ignite"); + A.notNull(endpointUrl, "Twitter Streaming API endpoint"); + + A.ensure(getSingleTupleExtractor() != null || getMultipleTupleExtractor() != null, "Twitter extractor"); + + String followParam = apiParams.get(SITE_USER_ID_KEY); + + A.ensure(followParam != null && followParam.matches("^(\\d+,? ?)+$"), + "Site streaming endpoint must provide 'follow' param with value as comma separated numbers"); + } + + /** + * @param tweetQueue Tweet queue. + * @return Client. + */ + protected Client getClient(BlockingQueue<String> tweetQueue) { + StreamingEndpoint endpoint; + + HttpHosts hosts; + + switch (endpointUrl.toLowerCase()) { + case StatusesFilterEndpoint.PATH: + endpoint = new StatusesFilterEndpoint(); + + hosts = HttpHosts.STREAM_HOST; + + break; + case StatusesFirehoseEndpoint.PATH: + endpoint = new StatusesFirehoseEndpoint(); + + hosts = HttpHosts.STREAM_HOST; + + break; + case StatusesSampleEndpoint.PATH: + endpoint = new StatusesSampleEndpoint(); + + hosts = HttpHosts.STREAM_HOST; + + break; + case UserstreamEndpoint.PATH: + endpoint = new UserstreamEndpoint(); + + hosts = HttpHosts.USERSTREAM_HOST; + + break; + case SitestreamEndpoint.PATH: + String follow = apiParams.remove(SITE_USER_ID_KEY); + + List<Long> followers = Lists.newArrayList(); + + for (String follower : Splitter.on(',').trimResults().omitEmptyStrings().split(follow)) + followers.add(Long.valueOf(follower)); + + endpoint = new SitestreamEndpoint(followers); + + hosts = HttpHosts.SITESTREAM_HOST; + + break; + default: + endpoint = new DefaultStreamingEndpoint(endpointUrl, HttpConstants.HTTP_GET, false); + + hosts = HttpHosts.STREAM_HOST; + + } + + for (Map.Entry<String, String> entry : apiParams.entrySet()) { + endpoint.addPostParameter(entry.getKey(), entry.getValue()); + } + + return buildClient(tweetQueue, hosts, endpoint); + } + + /** + * @param tweetQueue tweet Queue. + * @param hosts Hostes. + * @param endpoint Endpoint. + * @return Client. + */ + protected Client buildClient(BlockingQueue<String> tweetQueue, HttpHosts hosts, StreamingEndpoint endpoint) { + Authentication authentication = new OAuth1(oAuthSettings.getConsumerKey(), oAuthSettings.getConsumerSecret(), + oAuthSettings.getAccessToken(), oAuthSettings.getAccessTokenSecret()); + + ClientBuilder builder = new ClientBuilder() + .name("Ignite-Twitter-Client") + .hosts(hosts) + .authentication(authentication) + .endpoint(endpoint) + .processor(new StringDelimitedProcessor(tweetQueue)); + + return builder.build(); + } + + /** + * Sets API Params. + * + * @param apiParams API Params. + */ + public void setApiParams(Map<String, String> apiParams) { + this.apiParams = apiParams; + } + + /** + * Sets Endpoint URL. + * + * @param endpointUrl Endpoint URL. + */ + public void setEndpointUrl(String endpointUrl) { + this.endpointUrl = endpointUrl; + } + + /** + * Sets Buffer capacity. + * + * @param bufferCapacity Buffer capacity. + */ + public void setBufferCapacity(Integer bufferCapacity) { + this.bufferCapacity = bufferCapacity; + } + + /** + * Sets Threads count. + * + * @param threadsCount Threads count. + */ + public void setThreadsCount(int threadsCount) { + this.threadsCount = threadsCount; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/a39681ef/modules/twitter/src/test/java/org/apache/ignite/stream/twitter/IgniteTwitterStreamerTest.java ---------------------------------------------------------------------- diff --git a/modules/twitter/src/test/java/org/apache/ignite/stream/twitter/IgniteTwitterStreamerTest.java b/modules/twitter/src/test/java/org/apache/ignite/stream/twitter/IgniteTwitterStreamerTest.java new file mode 100644 index 0000000..4ba07ff --- /dev/null +++ b/modules/twitter/src/test/java/org/apache/ignite/stream/twitter/IgniteTwitterStreamerTest.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.stream.twitter; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.twitter.hbc.core.HttpHosts; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.IgniteException; +import org.apache.ignite.events.CacheEvent; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Rule; +import twitter4j.Status; +import twitter4j.TwitterException; +import twitter4j.TwitterObjectFactory; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; +import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_PUT; + +/** + * Test for {@link TwitterStreamer}. Tests Public Status streaming API https://dev.twitter.com/streaming/public. + */ +public class IgniteTwitterStreamerTest extends GridCommonAbstractTest { + + /** Cache entries count. */ + private static final int CACHE_ENTRY_COUNT = 100; + + /** Mocked api in embedded server. */ + private static final String MOCK_TWEET_PATH = "/tweet/mock"; + + /** Sample tweet. */ + private static final String tweet = "{\"id\":647375831971590144,\"text\":\"sample tweet to test streamer\"}\n"; + + /** Constructor. */ + public IgniteTwitterStreamerTest() { + super(true); + } + + /** Embedded mock HTTP server's for Twitter API rule. */ + @Rule + public final WireMockRule wireMockRule = new WireMockRule(); + + /** Embedded mock HTTP server for Twitter API. */ + public final WireMockServer mockServer = new WireMockServer(); //Starts server on 8080 port. + + /** {@inheritDoc} */ + @Override protected long getTestTimeout() { + return 10_000; + } + + /** {@inheritDoc} */ + @Override public void beforeTest() throws Exception { + grid().getOrCreateCache(defaultCacheConfiguration()); + + mockServer.start(); + + stubFor(get(urlMatching("/1.1" + MOCK_TWEET_PATH + ".*")).willReturn(aResponse(). + withHeader("Content-Type", "text/plain").withBody(tweet.length() + "\n" + tweet))); + } + + /** {@inheritDoc} */ + public void afterTest() throws Exception { + stopAllGrids(); + + mockServer.stop(); + } + + /** + * @throws Exception Test exception. + */ + public void testStatusesFilterEndpointOAuth1() throws Exception { + try (IgniteDataStreamer<Long, String> dataStreamer = grid().dataStreamer(null)) { + TwitterStreamerImpl streamer = newStreamerInstance(dataStreamer); + + Map<String, String> params = new HashMap<>(); + + params.put("track", "apache, twitter"); + params.put("follow", "3004445758");// @ApacheIgnite id. + + streamer.setApiParams(params); + streamer.setEndpointUrl(MOCK_TWEET_PATH); + streamer.setHosts(new HttpHosts("http://localhost:8080";)); + streamer.setThreadsCount(8); + + executeStreamer(streamer); + } + } + + /** + * @param streamer Twitter streamer. + * @throws InterruptedException Test exception. + * @throws TwitterException Test exception. + */ + private void executeStreamer(TwitterStreamer streamer) throws InterruptedException, TwitterException { + // Checking streaming. + + CacheListener listener = subscribeToPutEvents(); + + streamer.start(); + + try { + streamer.start(); + + fail("Successful start of already started Twitter Streamer"); + } + catch (IgniteException ex) { + // No-op. + } + + CountDownLatch latch = listener.getLatch(); + + // Enough tweets was handled in 10 seconds. Limited by test's timeout. + latch.await(); + + unsubscribeToPutEvents(listener); + + streamer.stop(); + + try { + streamer.stop(); + + fail("Successful stop of already stopped Twitter Streamer"); + } + catch (IgniteException ex) { + // No-op. + } + + // Checking cache content after streaming finished. + + Status status = TwitterObjectFactory.createStatus(tweet); + + IgniteCache<Long, String> cache = grid().cache(null); + + String cachedValue = cache.get(status.getId()); + + // Tweet successfully put to cache. + assertTrue(cachedValue != null && cachedValue.equals(status.getText())); + + // Same tweets does not produce duplicate entries. + assertTrue(cache.size() == 1); + } + + /** + * @return Cache listener. + */ + private CacheListener subscribeToPutEvents() { + Ignite ignite = grid(); + + // Listen to cache PUT events and expect as many as messages as test data items. + CacheListener listener = new CacheListener(); + + ignite.events(ignite.cluster().forCacheNodes(null)).localListen(listener, EVT_CACHE_OBJECT_PUT); + + return listener; + } + + /** + * @param listener Cache listener. + */ + private void unsubscribeToPutEvents(CacheListener listener) { + Ignite ignite = grid(); + + ignite.events(ignite.cluster().forCacheNodes(null)).stopLocalListen(listener, EVT_CACHE_OBJECT_PUT); + } + + /** + * @param dataStreamer Ignite Data Streamer. + * @return Twitter Streamer. + */ + private TwitterStreamerImpl newStreamerInstance(IgniteDataStreamer<Long, String> dataStreamer) { + OAuthSettings oAuthSettings = new OAuthSettings("<dummy>", "<dummy>", "<dummy>", "<dummy>"); + + TwitterStreamerImpl streamer = new TwitterStreamerImpl(oAuthSettings); + + streamer.setIgnite(grid()); + streamer.setStreamer(dataStreamer); + + dataStreamer.allowOverwrite(true); + dataStreamer.autoFlushFrequency(10); + + return streamer; + } + + /** + * Listener. + */ + private class CacheListener implements IgnitePredicate<CacheEvent> { + + /** */ + private final CountDownLatch latch = new CountDownLatch(CACHE_ENTRY_COUNT); + + /** + * @return Latch. + */ + public CountDownLatch getLatch() { + return latch; + } + + /** + * @param evt Cache Event. + * @return {@code true}. + */ + @Override + public boolean apply(CacheEvent evt) { + latch.countDown(); + + return true; + } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/a39681ef/modules/twitter/src/test/java/org/apache/ignite/stream/twitter/IgniteTwitterStreamerTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/twitter/src/test/java/org/apache/ignite/stream/twitter/IgniteTwitterStreamerTestSuite.java b/modules/twitter/src/test/java/org/apache/ignite/stream/twitter/IgniteTwitterStreamerTestSuite.java new file mode 100644 index 0000000..b458bed --- /dev/null +++ b/modules/twitter/src/test/java/org/apache/ignite/stream/twitter/IgniteTwitterStreamerTestSuite.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.stream.twitter; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * Twitter streamer tests. + */ +@RunWith(Suite.class) [email protected]({ + IgniteTwitterStreamerTest.class +}) +public class IgniteTwitterStreamerTestSuite { + +} http://git-wip-us.apache.org/repos/asf/ignite/blob/a39681ef/modules/twitter/src/test/java/org/apache/ignite/stream/twitter/TwitterStreamerImpl.java ---------------------------------------------------------------------- diff --git a/modules/twitter/src/test/java/org/apache/ignite/stream/twitter/TwitterStreamerImpl.java b/modules/twitter/src/test/java/org/apache/ignite/stream/twitter/TwitterStreamerImpl.java new file mode 100644 index 0000000..3c047e0 --- /dev/null +++ b/modules/twitter/src/test/java/org/apache/ignite/stream/twitter/TwitterStreamerImpl.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.stream.twitter; + +import com.twitter.hbc.core.Client; +import com.twitter.hbc.core.HttpHosts; +import com.twitter.hbc.core.endpoint.StreamingEndpoint; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiTuple; +import org.apache.ignite.stream.StreamSingleTupleExtractor; +import twitter4j.Status; +import twitter4j.TwitterException; +import twitter4j.TwitterObjectFactory; + +/** + * Long, String (With Mocked server support) implementation of TwitterStreamer. + */ +public class TwitterStreamerImpl extends TwitterStreamer<Long, String> { + /** Mocked server support. */ + HttpHosts hosts; + + /** + * @param oAuthSettings OAuth Settings + */ + public TwitterStreamerImpl(OAuthSettings oAuthSettings) { + super(oAuthSettings); + + setSingleTupleExtractor(new TwitterStreamSingleTupleExtractorImpl()); + } + + /** + * @param hosts hosts. + */ + public void setHosts(HttpHosts hosts) { + this.hosts = hosts; + } + + /** {@inheritDoc} */ + @Override protected Client buildClient(BlockingQueue<String> tweetQueue, HttpHosts hosts, + StreamingEndpoint endpoint) { + return super.buildClient(tweetQueue, this.hosts, endpoint); + } + + /** + * Long, String Tweet Single Tuple Extractor. + */ + class TwitterStreamSingleTupleExtractorImpl implements StreamSingleTupleExtractor<String, Long, String> { + @Override public Map.Entry<Long, String> extract(String tweet) { + try { + Status status = TwitterObjectFactory.createStatus(tweet); + + return new IgniteBiTuple<>(status.getId(), status.getText()); + } + catch (TwitterException e) { + U.error(log, e); + + return null; + } + } + } + +} http://git-wip-us.apache.org/repos/asf/ignite/blob/a39681ef/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index b9c51b2..1008981 100644 --- a/pom.xml +++ b/pom.xml @@ -76,6 +76,7 @@ <module>modules/flume</module> <module>modules/yarn</module> <module>modules/jms11</module> + <module>modules/twitter</module> <module>modules/mqtt</module> <module>modules/zookeeper</module> <module>modules/camel</module>
