Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Steven Jacobs has submitted this change and it was merged.
Change subject: Add user-stream for Twitter Adaptor
..
Add user-stream for Twitter Adaptor
1. Add user-stream option for Twitter Adaptor
2. Refactor part of TwitterRecordReaderFactory
3. To create a user-stream feed, using following ddl:
create feed TwitterFeed using twitter_user_stream(
("format"="twitter-status"),
("type-name"="Tweet"),
...
// rest is same as push feed
Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1272
Reviewed-by: Steven Jacobs
Tested-by: Jenkins
Integration-Tests: Jenkins
---
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TwitterUtil.java
5 files changed, 251 insertions(+), 76 deletions(-)
Approvals:
Steven Jacobs: Looks good to me, approved
Jenkins: Verified; Verified
Objections:
Jenkins: Violations found
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
index 9ead8a9..bd7 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
@@ -26,6 +26,8 @@
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.input.record.GenericRecord;
import org.apache.asterix.external.util.FeedLogManager;
+import org.apache.asterix.external.util.TwitterUtil;
+import twitter4j.DirectMessage;
import twitter4j.FilterQuery;
import twitter4j.StallWarning;
import twitter4j.Status;
@@ -33,6 +35,9 @@
import twitter4j.StatusListener;
import twitter4j.TwitterObjectFactory;
import twitter4j.TwitterStream;
+import twitter4j.User;
+import twitter4j.UserList;
+import twitter4j.UserStreamListener;
public class TwitterPushRecordReader implements IRecordReader {
private LinkedBlockingQueue inputQ;
@@ -40,20 +45,32 @@
private GenericRecord record;
private boolean closed = false;
-public TwitterPushRecordReader(TwitterStream twitterStream, FilterQuery
query) {
-record = new GenericRecord<>();
-inputQ = new LinkedBlockingQueue<>();
-this.twitterStream =
twitterStream;//TwitterUtil.getTwitterStream(configuration);
-this.twitterStream.addListener(new TweetListener(inputQ));
+public TwitterPushRecordReader(TwitterStream twitterStream,
TwitterUtil.TweetListener tweetListener,
+FilterQuery query) {
+init(twitterStream);
+tweetListener.setInputQ(inputQ);
+this.twitterStream.addListener(tweetListener);
this.twitterStream.filter(query);
}
-public TwitterPushRecordReader(TwitterStream twitterStream) {
+public TwitterPushRecordReader(TwitterStream twitterStream,
TwitterUtil.TweetListener tweetListener) {
+init(twitterStream);
+tweetListener.setInputQ(inputQ);
+this.twitterStream.addListener(tweetListener);
+twitterStream.sample();
+}
+
+public TwitterPushRecordReader(TwitterStream twitterStream,
TwitterUtil.UserTweetsListener tweetListener) {
+init(twitterStream);
+tweetListener.setInputQ(inputQ);
+this.twitterStream.addListener(tweetListener);
+twitterStream.user();
+}
+
+private void init(TwitterStream twitterStream) {
record = new GenericRecord<>();
inputQ = new LinkedBlockingQueue<>();
-this.twitterStream = twitterStream;//
-this.twitterStream.addListener(new TweetListener(inputQ));
-twitterStream.sample();
+this.twitterStream = twitterStream;
}
@Override
@@ -89,46 +106,6 @@
return false;
}
return true;
-}
-
-private class TweetListener implements StatusListener {
-
-private LinkedBlockingQueue inputQ;
-
-public TweetListener(LinkedBlockingQueue inputQ) {
-this.inputQ = inputQ;
-}
-
-@Override
-public void onStatus(Status tweet) {
-String jsonTweet = TwitterObjectFactory.getRawJSON(tweet);
-inputQ.add(
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Jenkins has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 3: Integration-Tests+1 Integration Tests Successful https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/989/ : SUCCESS -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 3 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Steven Jacobs Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Xikui Wang Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Jenkins has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 3: Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/989/ -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 3 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Steven Jacobs Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Xikui Wang Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Steven Jacobs has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 3: Code-Review+2 -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 3 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Steven Jacobs Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Xikui Wang Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Jenkins has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 3: Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/3098/ -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 3 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Steven Jacobs Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Xikui Wang Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Hello Steven Jacobs, Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/1272
to look at the new patch set (#3).
Change subject: Add user-stream for Twitter Adaptor
..
Add user-stream for Twitter Adaptor
1. Add user-stream option for Twitter Adaptor
2. Refactor part of TwitterRecordReaderFactory
3. To create a user-stream feed, using following ddl:
create feed TwitterFeed using twitter_user_stream(
("format"="twitter-status"),
("type-name"="Tweet"),
...
// rest is same as push feed
Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838
---
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TwitterUtil.java
5 files changed, 251 insertions(+), 76 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/72/1272/3
--
To view, visit https://asterix-gerrit.ics.uci.edu/1272
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838
Gerrit-PatchSet: 3
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang
Gerrit-Reviewer: Jenkins
Gerrit-Reviewer: Steven Jacobs
Gerrit-Reviewer: Till Westmann
Gerrit-Reviewer: Xikui Wang
Gerrit-Reviewer: abdullah alamoudi
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Steven Jacobs has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 2: Code-Review+2 -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Steven Jacobs Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Xikui Wang Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Steven Jacobs has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 2: +2 -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Steven Jacobs Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Xikui Wang Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Xikui Wang has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 2: (1 comment) Yes. The user of twitter feed needs to apply for Twitter API using certain twitter account. That's the account they are monitoring. :) https://asterix-gerrit.ics.uci.edu/#/c/1272/2/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java File asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java: Line 119: ? new TwitterPushRecordReader(TwitterUtil.getTwitterStream(configuration), > I'm guessing this doesn't get closed until the feed is dropped? Yes. Until the feed is stopped or dropped. -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Steven Jacobs Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Xikui Wang Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Steven Jacobs has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 2: (3 comments) Oh, so this is something that is already sent in the "push feed?" https://asterix-gerrit.ics.uci.edu/#/c/1272/2/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java File asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java: Line 68: } > Yeah. Basically the only part that could be shrunk is the tweetList Okay, let's just leave as it is then. https://asterix-gerrit.ics.uci.edu/#/c/1272/2/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java File asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java: Line 119: ? new TwitterPushRecordReader(TwitterUtil.getTwitterStream(configuration), > BLOCKER SonarQube violation: I'm guessing this doesn't get closed until the feed is dropped? Line 121: : new TwitterPushRecordReader(TwitterUtil.getTwitterStream(configuration), > BLOCKER SonarQube violation: I'm guessing this doesn't get closed until the feed is dropped? -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Steven Jacobs Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Xikui Wang Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Xikui Wang has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 2: (1 comment) The user whose access token is used in the feed creation, is the one that we are monitoring. https://asterix-gerrit.ics.uci.edu/#/c/1272/2/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java File asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java: Line 68: } > Can we extract the common code from these three constructors and move it to Yeah. Basically the only part that could be shrunk is the tweetListener.setInputQ(inputQ). But the tweetListener(s) are not the same in them, should I cast all to be tweetListener? -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Steven Jacobs Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: Xikui Wang Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Steven Jacobs has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 2: Also, the commit message doesn't show how to pass the user we want to listen to. -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Steven Jacobs Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: No
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Steven Jacobs has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 2: (1 comment) https://asterix-gerrit.ics.uci.edu/#/c/1272/2/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java File asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java: Line 68: } Can we extract the common code from these three constructors and move it to init? -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-Reviewer: Steven Jacobs Gerrit-Reviewer: Till Westmann Gerrit-Reviewer: abdullah alamoudi Gerrit-HasComments: Yes
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Jenkins has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 2: Integration-Tests+1 Integration Tests Successful https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/904/ : SUCCESS -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-HasComments: No
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Jenkins has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 2: Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/904/ -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-HasComments: No
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Jenkins has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 2: Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/2994/ -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 2 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-HasComments: No
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Hello Jenkins,
I'd like you to reexamine a change. Please visit
https://asterix-gerrit.ics.uci.edu/1272
to look at the new patch set (#2).
Change subject: Add user-stream for Twitter Adaptor
..
Add user-stream for Twitter Adaptor
1. Add user-stream option for Twitter Adaptor
2. Refactor part of TwitterRecordReaderFactory
3. To create a user-stream feed, using following ddl:
create feed TwitterFeed using twitter_user_stream(
("format"="twitter-status"),
("type-name"="Tweet"),
...
// rest is same as push feed
Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838
---
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TwitterUtil.java
5 files changed, 251 insertions(+), 76 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/72/1272/2
--
To view, visit https://asterix-gerrit.ics.uci.edu/1272
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newpatchset
Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838
Gerrit-PatchSet: 2
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Xikui Wang
Gerrit-Reviewer: Jenkins
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Jenkins has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 1: Integration-Tests+1 Integration Tests Successful https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/896/ : SUCCESS -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-HasComments: No
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Jenkins has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 1: Integration Tests Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/896/ -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-HasComments: No
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Jenkins has posted comments on this change. Change subject: Add user-stream for Twitter Adaptor .. Patch Set 1: Build Started https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-notopic/2985/ -- To view, visit https://asterix-gerrit.ics.uci.edu/1272 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: comment Gerrit-Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838 Gerrit-PatchSet: 1 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Xikui Wang Gerrit-Reviewer: Jenkins Gerrit-HasComments: No
Change in asterixdb[master]: Add user-stream for Twitter Adaptor
Xikui Wang has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1272
Change subject: Add user-stream for Twitter Adaptor
..
Add user-stream for Twitter Adaptor
1. Add user-stream option for Twitter Adaptor
2. Refactor part of TwitterRecordReaderFactory
3. To create a user-stream feed, using following ddl:
create feed TwitterFeed using twitter_user_stream(
("format"="twitter-status"),
("type-name"="Tweet"),
...
// rest is same as push feed
Change-Id: I99cdd4cb667306d378317616f9811dfce3e6d838
---
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterRecordReaderFactory.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TwitterUtil.java
5 files changed, 278 insertions(+), 93 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/72/1272/1
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
index 9ead8a9..c296bc6 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/twitter/TwitterPushRecordReader.java
@@ -26,6 +26,8 @@
import org.apache.asterix.external.dataflow.AbstractFeedDataFlowController;
import org.apache.asterix.external.input.record.GenericRecord;
import org.apache.asterix.external.util.FeedLogManager;
+import org.apache.asterix.external.util.TwitterUtil;
+import twitter4j.DirectMessage;
import twitter4j.FilterQuery;
import twitter4j.StallWarning;
import twitter4j.Status;
@@ -33,27 +35,43 @@
import twitter4j.StatusListener;
import twitter4j.TwitterObjectFactory;
import twitter4j.TwitterStream;
+import twitter4j.User;
+import twitter4j.UserList;
+import twitter4j.UserStreamListener;
public class TwitterPushRecordReader implements IRecordReader {
private LinkedBlockingQueue inputQ;
private TwitterStream twitterStream;
private GenericRecord record;
+private StatusListener tweetListener;
private boolean closed = false;
-public TwitterPushRecordReader(TwitterStream twitterStream, FilterQuery
query) {
-record = new GenericRecord<>();
-inputQ = new LinkedBlockingQueue<>();
-this.twitterStream =
twitterStream;//TwitterUtil.getTwitterStream(configuration);
-this.twitterStream.addListener(new TweetListener(inputQ));
+public TwitterPushRecordReader(TwitterStream twitterStream,
TwitterUtil.TweetListener tweetListener,
+FilterQuery query) {
+init(twitterStream);
+tweetListener.setInputQ(inputQ);
+this.twitterStream.addListener(tweetListener);
this.twitterStream.filter(query);
}
-public TwitterPushRecordReader(TwitterStream twitterStream) {
+public TwitterPushRecordReader(TwitterStream twitterStream,
TwitterUtil.TweetListener tweetListener) {
+init(twitterStream);
+tweetListener.setInputQ(inputQ);
+this.twitterStream.addListener(tweetListener);
+twitterStream.sample();
+}
+
+public TwitterPushRecordReader(TwitterStream twitterStream,
TwitterUtil.UserTweetsListener tweetListener) {
+init(twitterStream);
+tweetListener.setInputQ(inputQ);
+this.twitterStream.addListener(tweetListener);
+twitterStream.user();
+}
+
+private void init(TwitterStream twitterStream) {
record = new GenericRecord<>();
inputQ = new LinkedBlockingQueue<>();
-this.twitterStream = twitterStream;//
-this.twitterStream.addListener(new TweetListener(inputQ));
-twitterStream.sample();
+this.twitterStream = twitterStream;
}
@Override
@@ -89,46 +107,6 @@
return false;
}
return true;
-}
-
-private class TweetListener implements StatusListener {
-
-private LinkedBlockingQueue inputQ;
-
-public TweetListener(LinkedBlockingQueue inputQ) {
-this.inputQ = inputQ;
-}
-
-@Override
-public void onStatus(Status tweet) {
-String jsonTweet = TwitterObjectFactory.getRawJSON(tweet);
-inputQ.add(jsonTweet);
-}
-
-@Override
-public void onExcept
