Repository: incubator-streams-examples Updated Branches: refs/heads/master 34c1a7be2 -> 717e250c6
making ITs more flexible - but now must use -DargLine=â-Dconfig.file=<>â that imports database docker property files Project: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/commit/717e250c Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/tree/717e250c Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/diff/717e250c Branch: refs/heads/master Commit: 717e250c6786f7f17a0f40d10301d27587d51fbb Parents: 34c1a7b Author: Steve Blackmon <sblack...@apache.org> Authored: Tue Oct 18 22:49:30 2016 -0500 Committer: Steve Blackmon <sblack...@apache.org> Committed: Tue Oct 18 22:49:30 2016 -0500 ---------------------------------------------------------------------- ...linkTwitterFollowingPipelineFollowersIT.conf | 20 +++ .../FlinkTwitterFollowingPipelineFriendsIT.conf | 19 +++ .../resources/FlinkTwitterPostsPipelineIT.conf | 15 ++ .../FlinkTwitterSpritzerPipelineIT.conf | 15 ++ local/elasticsearch-hdfs/pom.xml | 13 +- .../example/test/ElasticsearchHdfsIT.java | 12 +- .../example/test/HdfsElasticsearchIT.java | 12 +- local/elasticsearch-reindex/pom.xml | 14 +- .../test/ElasticsearchReindexChildIT.java | 12 +- .../example/test/ElasticsearchReindexIT.java | 7 +- .../test/ElasticsearchReindexParentIT.java | 7 +- local/mongo-elasticsearch-sync/pom.xml | 13 +- .../example/test/MongoElasticsearchSyncIT.java | 18 +-- local/twitter-follow-neo4j/pom.xml | 6 +- .../test/TwitterHistoryElasticsearchIT.java | 12 +- local/twitter-userstream-elasticsearch/pom.xml | 2 +- .../test/TwitterUserstreamElasticsearchIT.java | 16 +-- pom.xml | 139 ++++++++++++++++++- 18 files changed, 220 insertions(+), 132 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/flink/flink-twitter-collection/src/site/resources/FlinkTwitterFollowingPipelineFollowersIT.conf ---------------------------------------------------------------------- diff --git a/flink/flink-twitter-collection/src/site/resources/FlinkTwitterFollowingPipelineFollowersIT.conf b/flink/flink-twitter-collection/src/site/resources/FlinkTwitterFollowingPipelineFollowersIT.conf new file mode 100644 index 0000000..3e922ab --- /dev/null +++ b/flink/flink-twitter-collection/src/site/resources/FlinkTwitterFollowingPipelineFollowersIT.conf @@ -0,0 +1,20 @@ +source { + fields = ["ID"] + scheme = file + path = "target/test-classes" + readerPath = "asf.txt" +} +destination { + fields = ["DOC"] + scheme = file + path = "target/test-classes" + writerPath = "FlinkTwitterFollowingPipelineFollowersIT" +} +twitter { + endpoint = followers + ids_only = true + max_items = 5000 +} +providerWaitMs = 1000 +local = true +test = true http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/flink/flink-twitter-collection/src/site/resources/FlinkTwitterFollowingPipelineFriendsIT.conf ---------------------------------------------------------------------- diff --git a/flink/flink-twitter-collection/src/site/resources/FlinkTwitterFollowingPipelineFriendsIT.conf b/flink/flink-twitter-collection/src/site/resources/FlinkTwitterFollowingPipelineFriendsIT.conf new file mode 100644 index 0000000..038a8dc --- /dev/null +++ b/flink/flink-twitter-collection/src/site/resources/FlinkTwitterFollowingPipelineFriendsIT.conf @@ -0,0 +1,19 @@ +source { + fields = ["ID"] + scheme = file + path = "target/test-classes" + readerPath = "asf.txt" +} +destination { + fields = ["DOC"] + scheme = file + path = "target/test-classes" + writerPath = "FlinkTwitterFollowingPipelineFriendsIT" +} +twitter { + endpoint = friends + ids_only = true +} +providerWaitMs = 1000 +local = true +test = true http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/flink/flink-twitter-collection/src/site/resources/FlinkTwitterPostsPipelineIT.conf ---------------------------------------------------------------------- diff --git a/flink/flink-twitter-collection/src/site/resources/FlinkTwitterPostsPipelineIT.conf b/flink/flink-twitter-collection/src/site/resources/FlinkTwitterPostsPipelineIT.conf new file mode 100644 index 0000000..6954113 --- /dev/null +++ b/flink/flink-twitter-collection/src/site/resources/FlinkTwitterPostsPipelineIT.conf @@ -0,0 +1,15 @@ +source { + fields = ["ID"] + scheme = file + path = "target/test-classes" + readerPath = "asf.txt" +} +destination { + fields = ["DOC"] + scheme = file + path = "target/test-classes" + writerPath = "FlinkTwitterPostsPipelineIT" +} +providerWaitMs = 1000 +local = true +test = true http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/flink/flink-twitter-collection/src/site/resources/FlinkTwitterSpritzerPipelineIT.conf ---------------------------------------------------------------------- diff --git a/flink/flink-twitter-collection/src/site/resources/FlinkTwitterSpritzerPipelineIT.conf b/flink/flink-twitter-collection/src/site/resources/FlinkTwitterSpritzerPipelineIT.conf new file mode 100644 index 0000000..fec4769 --- /dev/null +++ b/flink/flink-twitter-collection/src/site/resources/FlinkTwitterSpritzerPipelineIT.conf @@ -0,0 +1,15 @@ +destination { + fields = ["DOC"] + scheme = file + path = "target/test-classes" + writerPath = "FlinkTwitterSpritzerPipelineIT" +} +twitter { + endpoint = sample + track = [ + "data" + ] +} +providerWaitMs = 1000 +local = true +test = true http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/local/elasticsearch-hdfs/pom.xml ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/pom.xml b/local/elasticsearch-hdfs/pom.xml index 99458b1..eb52506 100644 --- a/local/elasticsearch-hdfs/pom.xml +++ b/local/elasticsearch-hdfs/pom.xml @@ -86,7 +86,7 @@ </dependency> <dependency> <groupId>org.apache.streams</groupId> - <artifactId>streams-pojo</artifactId> + <artifactId>streams-schema-activitystreams</artifactId> <version>0.4-incubating-SNAPSHOT</version> <type>test-jar</type> </dependency> @@ -178,17 +178,6 @@ </testResource> </testResources> <plugins> - <plugin> - <artifactId>maven-clean-plugin</artifactId> - <configuration> - <filesets> - <fileset> - <directory>data</directory> - <followSymlinks>false</followSymlinks> - </fileset> - </filesets> - </configuration> - </plugin> <!-- This binary runs with logback --> <!-- Keep log4j out --> <plugin> http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/test/ElasticsearchHdfsIT.java ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/test/ElasticsearchHdfsIT.java b/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/test/ElasticsearchHdfsIT.java index 8e87f3a..86d932b 100644 --- a/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/test/ElasticsearchHdfsIT.java +++ b/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/test/ElasticsearchHdfsIT.java @@ -23,8 +23,6 @@ import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import com.typesafe.config.ConfigParseOptions; import org.apache.streams.config.ComponentConfigurator; -import org.apache.streams.config.StreamsConfiguration; -import org.apache.streams.config.StreamsConfigurator; import org.apache.streams.elasticsearch.ElasticsearchClientManager; import org.apache.streams.example.ElasticsearchHdfs; import org.apache.streams.example.ElasticsearchHdfsConfiguration; @@ -44,9 +42,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.Properties; import static junit.framework.TestCase.assertTrue; import static org.junit.Assert.assertNotEquals; @@ -72,12 +67,7 @@ public class ElasticsearchHdfsIT { File conf_file = new File("target/test-classes/ElasticsearchHdfsIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - Properties es_properties = new Properties(); - InputStream es_stream = new FileInputStream("elasticsearch.properties"); - es_properties.load(es_stream); - Config esProps = ConfigFactory.parseProperties(es_properties); - Config typesafe = testResourceConfig.withFallback(esProps).withFallback(reference).resolve(); - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(ElasticsearchHdfsConfiguration.class).detectConfiguration(typesafe); testClient = new ElasticsearchClientManager(testConfiguration.getSource()).getClient(); http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/test/HdfsElasticsearchIT.java ---------------------------------------------------------------------- diff --git a/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/test/HdfsElasticsearchIT.java b/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/test/HdfsElasticsearchIT.java index 4eb7fc0..35a32e7 100644 --- a/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/test/HdfsElasticsearchIT.java +++ b/local/elasticsearch-hdfs/src/test/java/org/apache/streams/example/test/HdfsElasticsearchIT.java @@ -23,8 +23,6 @@ import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import com.typesafe.config.ConfigParseOptions; import org.apache.streams.config.ComponentConfigurator; -import org.apache.streams.config.StreamsConfiguration; -import org.apache.streams.config.StreamsConfigurator; import org.apache.streams.elasticsearch.ElasticsearchClientManager; import org.apache.streams.example.HdfsElasticsearch; import org.apache.streams.example.HdfsElasticsearchConfiguration; @@ -46,9 +44,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.Properties; import static junit.framework.TestCase.assertTrue; import static org.junit.Assert.assertEquals; @@ -73,12 +68,7 @@ public class HdfsElasticsearchIT { File conf_file = new File("target/test-classes/HdfsElasticsearchIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - Properties es_properties = new Properties(); - InputStream es_stream = new FileInputStream("elasticsearch.properties"); - es_properties.load(es_stream); - Config esProps = ConfigFactory.parseProperties(es_properties); - Config typesafe = testResourceConfig.withFallback(esProps).withFallback(reference).resolve(); - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(HdfsElasticsearchConfiguration.class).detectConfiguration(typesafe); testClient = new ElasticsearchClientManager(testConfiguration.getDestination()).getClient(); http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/local/elasticsearch-reindex/pom.xml ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/pom.xml b/local/elasticsearch-reindex/pom.xml index 4678c42..3b0067a 100644 --- a/local/elasticsearch-reindex/pom.xml +++ b/local/elasticsearch-reindex/pom.xml @@ -82,7 +82,8 @@ </dependency> <dependency> <groupId>org.apache.streams</groupId> - <artifactId>streams-pojo</artifactId> + <artifactId>streams-schema-activitystreams</artifactId> + <version>0.4-incubating-SNAPSHOT</version> <type>test-jar</type> </dependency> <dependency> @@ -142,17 +143,6 @@ </testResource> </testResources> <plugins> - <plugin> - <artifactId>maven-clean-plugin</artifactId> - <configuration> - <filesets> - <fileset> - <directory>data</directory> - <followSymlinks>false</followSymlinks> - </fileset> - </filesets> - </configuration> - </plugin> <!-- This binary runs with logback --> <!-- Keep log4j out --> <plugin> http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexChildIT.java ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexChildIT.java b/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexChildIT.java index 47c8f51..631a084 100644 --- a/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexChildIT.java +++ b/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexChildIT.java @@ -23,8 +23,6 @@ import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import com.typesafe.config.ConfigParseOptions; import org.apache.streams.config.ComponentConfigurator; -import org.apache.streams.config.StreamsConfiguration; -import org.apache.streams.config.StreamsConfigurator; import org.apache.streams.elasticsearch.ElasticsearchClientManager; import org.apache.streams.example.ElasticsearchReindex; import org.apache.streams.example.ElasticsearchReindexConfiguration; @@ -44,9 +42,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.Properties; import static junit.framework.TestCase.assertTrue; import static org.junit.Assert.assertEquals; @@ -73,12 +68,7 @@ public class ElasticsearchReindexChildIT { File conf_file = new File("target/test-classes/ElasticsearchReindexChildIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - Properties es_properties = new Properties(); - InputStream es_stream = new FileInputStream("elasticsearch.properties"); - es_properties.load(es_stream); - Config esProps = ConfigFactory.parseProperties(es_properties); - Config typesafe = testResourceConfig.withFallback(esProps).withFallback(reference).resolve(); - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(ElasticsearchReindexConfiguration.class).detectConfiguration(typesafe); testClient = new ElasticsearchClientManager(testConfiguration.getSource()).getClient(); http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexIT.java ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexIT.java b/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexIT.java index 3fee0d7..9ef2573 100644 --- a/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexIT.java +++ b/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexIT.java @@ -73,12 +73,7 @@ public class ElasticsearchReindexIT { File conf_file = new File("target/test-classes/ElasticsearchReindexIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - Properties es_properties = new Properties(); - InputStream es_stream = new FileInputStream("elasticsearch.properties"); - es_properties.load(es_stream); - Config esProps = ConfigFactory.parseProperties(es_properties); - Config typesafe = testResourceConfig.withFallback(esProps).withFallback(reference).resolve(); - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(ElasticsearchReindexConfiguration.class).detectConfiguration(typesafe); testClient = new ElasticsearchClientManager(testConfiguration.getSource()).getClient(); http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexParentIT.java ---------------------------------------------------------------------- diff --git a/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexParentIT.java b/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexParentIT.java index fc80453..ccc73c6 100644 --- a/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexParentIT.java +++ b/local/elasticsearch-reindex/src/test/java/org/apache/streams/example/test/ElasticsearchReindexParentIT.java @@ -77,12 +77,7 @@ public class ElasticsearchReindexParentIT { File conf_file = new File("target/test-classes/ElasticsearchReindexParentIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - Properties es_properties = new Properties(); - InputStream es_stream = new FileInputStream("elasticsearch.properties"); - es_properties.load(es_stream); - Config esProps = ConfigFactory.parseProperties(es_properties); - Config typesafe = testResourceConfig.withFallback(esProps).withFallback(reference).resolve(); - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(ElasticsearchReindexConfiguration.class).detectConfiguration(typesafe); testClient = new ElasticsearchClientManager(testConfiguration.getSource()).getClient(); http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/local/mongo-elasticsearch-sync/pom.xml ---------------------------------------------------------------------- diff --git a/local/mongo-elasticsearch-sync/pom.xml b/local/mongo-elasticsearch-sync/pom.xml index 2a89e0d..435a40b 100644 --- a/local/mongo-elasticsearch-sync/pom.xml +++ b/local/mongo-elasticsearch-sync/pom.xml @@ -101,7 +101,7 @@ </dependency> <dependency> <groupId>org.apache.streams</groupId> - <artifactId>streams-pojo</artifactId> + <artifactId>streams-schema-activitystreams</artifactId> <version>0.4-incubating-SNAPSHOT</version> <type>test-jar</type> </dependency> @@ -176,17 +176,6 @@ </executions> </plugin> <plugin> - <artifactId>maven-clean-plugin</artifactId> - <configuration> - <filesets> - <fileset> - <directory>data</directory> - <followSymlinks>false</followSymlinks> - </fileset> - </filesets> - </configuration> - </plugin> - <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> </plugin> http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/local/mongo-elasticsearch-sync/src/test/java/org/apache/streams/example/test/MongoElasticsearchSyncIT.java ---------------------------------------------------------------------- diff --git a/local/mongo-elasticsearch-sync/src/test/java/org/apache/streams/example/test/MongoElasticsearchSyncIT.java b/local/mongo-elasticsearch-sync/src/test/java/org/apache/streams/example/test/MongoElasticsearchSyncIT.java index 47851f3..f10947b 100644 --- a/local/mongo-elasticsearch-sync/src/test/java/org/apache/streams/example/test/MongoElasticsearchSyncIT.java +++ b/local/mongo-elasticsearch-sync/src/test/java/org/apache/streams/example/test/MongoElasticsearchSyncIT.java @@ -44,14 +44,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.Properties; import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.*; /** * Test copying documents between two indexes on same cluster @@ -72,16 +67,7 @@ public class MongoElasticsearchSyncIT { File conf_file = new File("target/test-classes/MongoElasticsearchSyncIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - Properties mongo_properties = new Properties(); - InputStream mongo_stream = new FileInputStream("mongo.properties"); - mongo_properties.load(mongo_stream); - Config mongoProps = ConfigFactory.parseProperties(mongo_properties); - Properties es_properties = new Properties(); - InputStream es_stream = new FileInputStream("elasticsearch.properties"); - es_properties.load(es_stream); - Config esProps = ConfigFactory.parseProperties(es_properties); - Config typesafe = testResourceConfig.withFallback(mongoProps).withFallback(esProps).withFallback(reference).resolve(); - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(MongoElasticsearchSyncConfiguration.class).detectConfiguration(typesafe); testClient = new ElasticsearchClientManager(testConfiguration.getDestination()).getClient(); http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/local/twitter-follow-neo4j/pom.xml ---------------------------------------------------------------------- diff --git a/local/twitter-follow-neo4j/pom.xml b/local/twitter-follow-neo4j/pom.xml index e644c3c..d9b7053 100644 --- a/local/twitter-follow-neo4j/pom.xml +++ b/local/twitter-follow-neo4j/pom.xml @@ -26,8 +26,8 @@ </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>twitter-follow-graph</artifactId> - <name>twitter-follow-graph</name> + <artifactId>twitter-follow-neo4j</artifactId> + <name>twitter-follow-neo4j</name> <description> Collects friend or follower connections for a set of twitter users to build a graph database in neo4j. @@ -215,7 +215,7 @@ <goal>unpack-dependencies</goal> </goals> <configuration> - <includeArtifactIds>streams-pojo</includeArtifactIds> + <includeArtifactIds>streams-schemas-activitystreams</includeArtifactIds> <includes>**/*.json</includes> <outputDirectory>${project.build.directory}/test-classes</outputDirectory> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/local/twitter-history-elasticsearch/src/test/java/org/apache/streams/example/test/TwitterHistoryElasticsearchIT.java ---------------------------------------------------------------------- diff --git a/local/twitter-history-elasticsearch/src/test/java/org/apache/streams/example/test/TwitterHistoryElasticsearchIT.java b/local/twitter-history-elasticsearch/src/test/java/org/apache/streams/example/test/TwitterHistoryElasticsearchIT.java index b0c9155..dc1b443 100644 --- a/local/twitter-history-elasticsearch/src/test/java/org/apache/streams/example/test/TwitterHistoryElasticsearchIT.java +++ b/local/twitter-history-elasticsearch/src/test/java/org/apache/streams/example/test/TwitterHistoryElasticsearchIT.java @@ -22,8 +22,6 @@ import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import com.typesafe.config.ConfigParseOptions; import org.apache.streams.config.ComponentConfigurator; -import org.apache.streams.config.StreamsConfiguration; -import org.apache.streams.config.StreamsConfigurator; import org.apache.streams.elasticsearch.ElasticsearchClientManager; import org.apache.streams.example.TwitterHistoryElasticsearch; import org.apache.streams.example.TwitterHistoryElasticsearchConfiguration; @@ -42,9 +40,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.Properties; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -68,12 +63,7 @@ public class TwitterHistoryElasticsearchIT { File conf_file = new File("target/test-classes/TwitterHistoryElasticsearchIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - Properties es_properties = new Properties(); - InputStream es_stream = new FileInputStream("elasticsearch.properties"); - es_properties.load(es_stream); - Config esProps = ConfigFactory.parseProperties(es_properties); - Config typesafe = testResourceConfig.withFallback(esProps).withFallback(reference).resolve(); - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(TwitterHistoryElasticsearchConfiguration.class).detectConfiguration(typesafe); testClient = new ElasticsearchClientManager(testConfiguration.getElasticsearch()).getClient(); http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/local/twitter-userstream-elasticsearch/pom.xml ---------------------------------------------------------------------- diff --git a/local/twitter-userstream-elasticsearch/pom.xml b/local/twitter-userstream-elasticsearch/pom.xml index 8e14913..559e1b9 100644 --- a/local/twitter-userstream-elasticsearch/pom.xml +++ b/local/twitter-userstream-elasticsearch/pom.xml @@ -192,7 +192,7 @@ <includes>**/*.json</includes> <outputDirectory>${project.build.directory}/test-classes</outputDirectory> <includeGroupIds>org.apache.streams</includeGroupIds> - <includeArtifactIds>streams-pojo</includeArtifactIds> + <includeArtifactIds>streams-schemas-activitystreams</includeArtifactIds> <includeTypes>test-jar</includeTypes> </configuration> <executions> http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/local/twitter-userstream-elasticsearch/src/test/java/org/apache/streams/example/test/TwitterUserstreamElasticsearchIT.java ---------------------------------------------------------------------- diff --git a/local/twitter-userstream-elasticsearch/src/test/java/org/apache/streams/example/test/TwitterUserstreamElasticsearchIT.java b/local/twitter-userstream-elasticsearch/src/test/java/org/apache/streams/example/test/TwitterUserstreamElasticsearchIT.java index 7ba9940..e2b5509 100644 --- a/local/twitter-userstream-elasticsearch/src/test/java/org/apache/streams/example/test/TwitterUserstreamElasticsearchIT.java +++ b/local/twitter-userstream-elasticsearch/src/test/java/org/apache/streams/example/test/TwitterUserstreamElasticsearchIT.java @@ -22,11 +22,9 @@ import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import com.typesafe.config.ConfigParseOptions; import org.apache.streams.config.ComponentConfigurator; -import org.apache.streams.config.StreamsConfiguration; -import org.apache.streams.config.StreamsConfigurator; import org.apache.streams.elasticsearch.ElasticsearchClientManager; -import org.apache.streams.example.TwitterUserstreamElasticsearchConfiguration; import org.apache.streams.example.TwitterUserstreamElasticsearch; +import org.apache.streams.example.TwitterUserstreamElasticsearchConfiguration; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; @@ -42,12 +40,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.Properties; -import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -70,12 +63,7 @@ public class TwitterUserstreamElasticsearchIT { File conf_file = new File("target/test-classes/TwitterUserstreamElasticsearchIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - Properties es_properties = new Properties(); - InputStream es_stream = new FileInputStream("elasticsearch.properties"); - es_properties.load(es_stream); - Config esProps = ConfigFactory.parseProperties(es_properties); - Config typesafe = testResourceConfig.withFallback(esProps).withFallback(reference).resolve(); - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(TwitterUserstreamElasticsearchConfiguration.class).detectConfiguration(typesafe); testClient = new ElasticsearchClientManager(testConfiguration.getElasticsearch()).getClient(); http://git-wip-us.apache.org/repos/asf/incubator-streams-examples/blob/717e250c/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 882aa13..8246cb4 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ <artifactId>streams-master</artifactId> <groupId>org.apache.streams</groupId> <version>0.4-incubating-SNAPSHOT</version> - </parent> + </parent> <artifactId>streams-examples</artifactId> <version>0.4-incubating-SNAPSHOT</version> @@ -48,8 +48,8 @@ <connection>scm:git:g...@github.com:apache/incubator-streams-examples.git</connection> <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-streams-examples.git</developerConnection> <url>scm:git:g...@github.com:apache/incubator-streams-examples.git</url> - <tag>HEAD</tag> - </scm> + <tag>HEAD</tag> + </scm> <developers> <developer> @@ -310,9 +310,9 @@ <resource>application.conf</resource> </transformer> <!--<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">--> - <!--<manifestEntries>--> - <!--<Main-Class>akka.Main</Main-Class>--> - <!--</manifestEntries>--> + <!--<manifestEntries>--> + <!--<Main-Class>akka.Main</Main-Class>--> + <!--</manifestEntries>--> <!--</transformer>--> </transformers> </configuration> @@ -388,5 +388,132 @@ </plugins> </build> </profile> + + <profile> + <id>dockerITs</id> + <activation> + <activeByDefault>false</activeByDefault> + <property> + <name>skipITs</name> + <value>false</value> + </property> + </activation> + <build> + <plugins> + <plugin> + <groupId>io.fabric8</groupId> + <artifactId>docker-maven-plugin</artifactId> + <version>${docker.plugin.version}</version> + <configuration combine.self="override"> + <watchInterval>500</watchInterval> + <logDate>default</logDate> + <verbose>true</verbose> + <autoPull>on</autoPull> + <images> + <image> + <name>elasticsearch:2.3.5</name> + <alias>elasticsearch</alias> + <run> + <namingStrategy>none</namingStrategy> + <ports> + <port>${es.http.host}:${es.http.port}:9200</port> + <port>${es.tcp.host}:${es.tcp.port}:9300</port> + </ports> + <portPropertyFile>elasticsearch.properties</portPropertyFile> + <wait> + <log>elasticsearch startup</log> + <http> + <url>http://${es.http.host}:${es.http.port}</url> + <method>GET</method> + <status>200</status> + </http> + <time>20000</time> + <kill>1000</kill> + <shutdown>500</shutdown> + <!--<tcp>--> + <!--<host>${es.transport.host}</host>--> + <!--<ports>--> + <!--<port>${es.transport.port}</port>--> + <!--</ports>--> + <!--</tcp>--> + </wait> + <log> + <enabled>true</enabled> + <date>default</date> + <color>cyan</color> + </log> + </run> + <watch> + <mode>none</mode> + </watch> + </image> + <image> + <name>mongo:3.2.0</name> + <alias>mongo</alias> + <run> + <namingStrategy>none</namingStrategy> + <ports> + <port>${mongo.tcp.host}:${mongo.tcp.port}:27017</port> + </ports> + <portPropertyFile>mongo.properties</portPropertyFile> + <log> + <enabled>true</enabled> + <date>default</date> + <color>cyan</color> + </log> + </run> + <watch> + <mode>none</mode> + </watch> + </image> + <image> + <name>neo4j</name> + <alias>graph</alias> + <run> + <env> + <NEO4J_AUTH>none</NEO4J_AUTH> + </env> + <namingStrategy>none</namingStrategy> + <ports> + <port>${neo4j.http.host}:${neo4j.http.port}:7474</port> + <port>${neo4j.tcp.host}:${neo4j.tcp.port}:7687</port> + </ports> + <portPropertyFile>neo4j.properties</portPropertyFile> + <wait> + <log>graph startup</log> + <http> + <url>http://${neo4j.http.host}:${neo4j.http.port}</url> + <method>GET</method> + <status>200</status> + </http> + <time>20000</time> + <kill>1000</kill> + <shutdown>500</shutdown> + <!--<tcp>--> + <!--<host>${es.transport.host}</host>--> + <!--<ports>--> + <!--<port>${es.transport.port}</port>--> + <!--</ports>--> + <!--</tcp>--> + </wait> + <log> + <enabled>true</enabled> + <date>default</date> + <color>cyan</color> + </log> + </run> + <watch> + <mode>none</mode> + </watch> + </image> + </images> + </configuration> + + </plugin> + + </plugins> + </build> + + </profile> </profiles> </project>
