abdullah alamoudi has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1290
Change subject: Move Twitter Firehose Datasource to Test Source Folder
......................................................................
Move Twitter Firehose Datasource to Test Source Folder
Change-Id: Iefe2130707012b8ce60f5dfac96635a1a515a076
---
M
asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_07/feeds_07.1.ddl.aql
M
asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
M
asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_09/feeds_09.1.ddl.aql
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_07/feeds_07.1.ddl.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_08/feeds_08.1.ddl.sqlpp
M
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_09/feeds_09.1.ddl.sqlpp
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
R
asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/DataGenerator.java
R
asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/TweetGenerator.java
R
asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
R
asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java
M
asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/DataGeneratorForSpatialIndexEvaluation.java
13 files changed, 25 insertions(+), 27 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/90/1290/1
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_07/feeds_07.1.ddl.aql
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_07/feeds_07.1.ddl.aql
index 70ea8d6..f3f8f7d 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_07/feeds_07.1.ddl.aql
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_07/feeds_07.1.ddl.aql
@@ -54,12 +54,12 @@
primary key id;
create feed SyntheticTweetFeed
-using twitter_firehose(
+using stream(
("duration"="5"),
("tps"="50"),
("type-name"="TweetMessageType"),
("format"="adm"),
-("reader-stream"="twitter_firehose"),
+("stream-source"="org.apache.asterix.external.input.stream.TwitterFirehoseStreamFactory"),
("tput-duration"="5"),
("dataverse-dataset"="feeds:SyntheticTweets"),
("mode"="controlled"));
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
index 658487b..c339563 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_08/feeds_08.1.ddl.aql
@@ -58,8 +58,9 @@
create index ngram_index on SyntheticTweets(message_text) type ngram(3);
create feed SyntheticTweetFeed
-using twitter_firehose
+using stream
(("duration"="5"),
+("stream-source"="org.apache.asterix.external.input.stream.TwitterFirehoseStreamFactory"),
("tps"="50"),
("type-name"="TweetMessageType"),
("tput-duration"="5"),
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_09/feeds_09.1.ddl.aql
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_09/feeds_09.1.ddl.aql
index 6714850..59385c4 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_09/feeds_09.1.ddl.aql
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/feeds_09/feeds_09.1.ddl.aql
@@ -56,12 +56,12 @@
create index message_text on SyntheticTweets(message_text) type btree;
create feed SyntheticTweetFeed
-using twitter_firehose
+using stream
(("duration"="5"),
("tps"="50"),
("tput-duration"="5"),
("type-name"="TweetMessageType"),
("dataverse-dataset"="feeds:SyntheticTweets"),
("format"="adm"),
-("reader-stream"="twitter_firehose"),
+("stream-source"="org.apache.asterix.external.input.stream.TwitterFirehoseStreamFactory"),
("mode"="controlled"));
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_07/feeds_07.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_07/feeds_07.1.ddl.sqlpp
index e3f3ae5..1f24192 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_07/feeds_07.1.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_07/feeds_07.1.ddl.sqlpp
@@ -45,12 +45,12 @@
create dataset SyntheticTweets(TweetMessageType) primary key id;
-create primary feed SyntheticTweetFeed using twitter_firehose(
+create primary feed SyntheticTweetFeed using stream(
(`duration`=`5`),
(`tps`=`50`),
(`type-name`=`TweetMessageType`),
(`format`=`adm`),
-(`reader-stream`=`twitter_firehose`),
+(`stream-source`=`org.apache.asterix.external.input.stream.TwitterFirehoseStreamFactory`),
(`tput-duration`=`5`),
(`dataverse-dataset`=`feeds:SyntheticTweets`),
(`mode`=`controlled`));
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_08/feeds_08.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_08/feeds_08.1.ddl.sqlpp
index a98b745..6311b8b 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_08/feeds_08.1.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_08/feeds_08.1.ddl.sqlpp
@@ -45,9 +45,10 @@
create dataset SyntheticTweets(TweetMessageType) primary key id;
create index ngram_index on SyntheticTweets (message_text) type ngram (3);
-create primary feed SyntheticTweetFeed using twitter_firehose (
+create primary feed SyntheticTweetFeed using stream (
(`duration`=`5`),
(`tps`=`50`),
+(`stream-source`=`org.apache.asterix.external.input.stream.TwitterFirehoseStreamFactory`),
(`type-name`=`TweetMessageType`),
(`tput-duration`=`5`),
(`dataverse-dataset`=`feeds:SyntheticTweets`),
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_09/feeds_09.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_09/feeds_09.1.ddl.sqlpp
index 1b1c780..71a0ca2 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_09/feeds_09.1.ddl.sqlpp
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/feeds/feeds_09/feeds_09.1.ddl.sqlpp
@@ -45,7 +45,7 @@
}
create dataset SyntheticTweets(TweetMessageType) primary key id;
create index message_text on SyntheticTweets (message_text) type btree;
-create primary feed SyntheticTweetFeed using twitter_firehose
((`duration`=`5`),
+create primary feed SyntheticTweetFeed using stream ((`duration`=`5`),
(`tps`=`50`),(`tput-duration`=`5`),(`type-name`=`TweetMessageType`),
(`dataverse-dataset`=`feeds:SyntheticTweets`),(`format`=`adm`),
-(`reader-stream`=`twitter_firehose`),(`mode`=`controlled`));
+(`stream-source`=`org.apache.asterix.external.input.stream.TwitterFirehoseStreamFactory`),(`mode`=`controlled`));
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
index ad11171..7ab6430 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/provider/DatasourceFactoryProvider.java
@@ -33,7 +33,6 @@
import
org.apache.asterix.external.input.stream.factory.LocalFSInputStreamFactory;
import
org.apache.asterix.external.input.stream.factory.SocketClientInputStreamFactory;
import
org.apache.asterix.external.input.stream.factory.SocketServerInputStreamFactory;
-import
org.apache.asterix.external.input.stream.factory.TwitterFirehoseStreamFactory;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
@@ -73,9 +72,6 @@
case ExternalDataConstants.STREAM_SOCKET_CLIENT:
streamSourceFactory = new SocketServerInputStreamFactory();
break;
- case ExternalDataConstants.ALIAS_TWITTER_FIREHOSE_ADAPTER:
- streamSourceFactory = new TwitterFirehoseStreamFactory();
- break;
default:
try {
streamSourceFactory = (IInputStreamFactory)
Class.forName(streamSource).newInstance();
@@ -102,8 +98,6 @@
case ExternalDataConstants.READER_PUSH_TWITTER:
case ExternalDataConstants.READER_PULL_TWITTER:
return new TwitterRecordReaderFactory();
- case ExternalDataConstants.ALIAS_TWITTER_FIREHOSE_ADAPTER:
- return new StreamRecordReaderFactory(new
TwitterFirehoseStreamFactory());
case ExternalDataConstants.ALIAS_SOCKET_ADAPTER:
case ExternalDataConstants.SOCKET:
return new StreamRecordReaderFactory(new
SocketServerInputStreamFactory());
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 7d27e45..e795730 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -174,7 +174,6 @@
public static final String ALIAS_LOCALFS_PUSH_ADAPTER = "push_localfs";
public static final String ALIAS_HDFS_ADAPTER = "hdfs";
public static final String ALIAS_SOCKET_ADAPTER = "socket_adapter";
- public static final String ALIAS_TWITTER_FIREHOSE_ADAPTER =
"twitter_firehose";
public static final String ALIAS_SOCKET_CLIENT_ADAPTER = "socket_client";
public static final String ALIAS_RSS_ADAPTER = "rss";
public static final String ALIAS_FILE_FEED_ADAPTER = "file_feed";
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/DataGenerator.java
b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/DataGenerator.java
similarity index 99%
rename from
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/DataGenerator.java
rename to
asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/DataGenerator.java
index 59c9a0f..f61c7d0 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/DataGenerator.java
+++
b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/DataGenerator.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.util;
+package org.apache.asterix.external.generator;
import java.nio.CharBuffer;
import java.util.ArrayList;
@@ -25,6 +25,9 @@
import java.util.List;
import java.util.Random;
+import org.apache.asterix.external.util.Datatypes;
+import org.apache.asterix.external.util.Datatypes.Tweet;
+
public class DataGenerator {
private RandomDateGenerator randDateGen;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TweetGenerator.java
b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/TweetGenerator.java
similarity index 93%
rename from
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TweetGenerator.java
rename to
asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/TweetGenerator.java
index b86c170..88ca4cc 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/TweetGenerator.java
+++
b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/generator/TweetGenerator.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.util;
+package org.apache.asterix.external.generator;
import java.io.IOException;
import java.io.OutputStream;
@@ -27,15 +27,15 @@
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.external.util.DataGenerator.InitializationInfo;
-import org.apache.asterix.external.util.DataGenerator.TweetMessage;
-import org.apache.asterix.external.util.DataGenerator.TweetMessageIterator;
+import org.apache.asterix.external.generator.DataGenerator.InitializationInfo;
+import org.apache.asterix.external.generator.DataGenerator.TweetMessage;
+import
org.apache.asterix.external.generator.DataGenerator.TweetMessageIterator;
public class TweetGenerator {
private static final Logger LOGGER =
Logger.getLogger(TweetGenerator.class.getName());
public static final String KEY_DURATION = "duration";
- public static final String KEY_TPS = "tps";
+ public static final String KEY_TPS = "tps";
public static final String KEY_VERBOSE = "verbose";
public static final String KEY_FIELDS = "fields";
public static final int INFINITY = 0;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
similarity index 98%
rename from
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
rename to
asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
index e2afd7b..dd98691 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
+++
b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseInputStream.java
@@ -29,7 +29,7 @@
import java.util.logging.Logger;
import org.apache.asterix.external.api.AsterixInputStream;
-import org.apache.asterix.external.util.TweetGenerator;
+import org.apache.asterix.external.generator.TweetGenerator;
import org.apache.hyracks.api.context.IHyracksTaskContext;
public class TwitterFirehoseInputStream extends AsterixInputStream {
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java
similarity index 96%
rename from
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
rename to
asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java
index abe67fd..77b8df1 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/TwitterFirehoseStreamFactory.java
+++
b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.external.input.stream.factory;
+package org.apache.asterix.external.input.stream;
import java.io.IOException;
import java.util.ArrayList;
@@ -67,7 +67,7 @@
count = Integer.parseInt(ingestionCardinalityParam);
}
- List<String> chosenLocations = new ArrayList<String>();
+ List<String> chosenLocations = new ArrayList<>();
String[] availableLocations = locations != null ? locations
:
ClusterStateManager.INSTANCE.getParticipantNodes().toArray(new String[] {});
for (int i = 0, k = 0; i < count; i++, k = (k + 1) %
availableLocations.length) {
diff --git
a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/DataGeneratorForSpatialIndexEvaluation.java
b/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/DataGeneratorForSpatialIndexEvaluation.java
index 93ffe59..763a4ae 100644
---
a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/DataGeneratorForSpatialIndexEvaluation.java
+++
b/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/DataGeneratorForSpatialIndexEvaluation.java
@@ -30,7 +30,7 @@
import java.util.List;
import java.util.Random;
-import org.apache.asterix.external.util.DataGenerator;
+import org.apache.asterix.external.generator.DataGenerator;
public class DataGeneratorForSpatialIndexEvaluation {
--
To view, visit https://asterix-gerrit.ics.uci.edu/1290
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Iefe2130707012b8ce60f5dfac96635a1a515a076
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <[email protected]>
