Repository: incubator-streams Updated Branches: refs/heads/master 5fd26a677 -> 095c082ff
making ITs more flexible - but now must use -DargLine=â-Dconfig.file=<>â that imports database docker property files Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/095c082f Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/095c082f Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/095c082f Branch: refs/heads/master Commit: 095c082ffadf05e7f7830d4e81a6231e3ffc88e3 Parents: 5fd26a6 Author: Steve Blackmon <sblack...@apache.org> Authored: Tue Oct 18 22:45:39 2016 -0500 Committer: Steve Blackmon <sblack...@apache.org> Committed: Tue Oct 18 22:45:39 2016 -0500 ---------------------------------------------------------------------- .../streams-persist-elasticsearch/pom.xml | 2 +- .../test/DatumFromMetadataProcessorIT.java | 16 +--------------- .../test/ElasticsearchParentChildUpdaterIT.java | 7 +------ .../test/ElasticsearchParentChildWriterIT.java | 18 +----------------- .../test/ElasticsearchPersistUpdaterIT.java | 7 +------ .../test/ElasticsearchPersistWriterIT.java | 18 ++---------------- streams-contrib/streams-persist-mongo/pom.xml | 2 +- .../apache/streams/mongo/test/MongoPersistIT.java | 13 +------------ 8 files changed, 9 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/095c082f/streams-contrib/streams-persist-elasticsearch/pom.xml ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-persist-elasticsearch/pom.xml b/streams-contrib/streams-persist-elasticsearch/pom.xml index 78f6e7f..e135e9d 100644 --- a/streams-contrib/streams-persist-elasticsearch/pom.xml +++ b/streams-contrib/streams-persist-elasticsearch/pom.xml @@ -290,7 +290,7 @@ <port>${es.http.host}:${es.http.port}:9200</port> <port>${es.tcp.host}:${es.tcp.port}:9300</port> </ports> - <portPropertyFile>${pom.basedir}/elasticsearch.properties</portPropertyFile> + <portPropertyFile>elasticsearch.properties</portPropertyFile> <wait> <log>elasticsearch startup</log> <http> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/095c082f/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/DatumFromMetadataProcessorIT.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/DatumFromMetadataProcessorIT.java b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/DatumFromMetadataProcessorIT.java index 8d8bb90..c81d183 100644 --- a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/DatumFromMetadataProcessorIT.java +++ b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/DatumFromMetadataProcessorIT.java @@ -18,32 +18,23 @@ package org.apache.streams.elasticsearch.test; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import com.typesafe.config.ConfigParseOptions; import org.apache.commons.lang.SerializationUtils; import org.apache.streams.config.ComponentConfigurator; -import org.apache.streams.config.StreamsConfiguration; -import org.apache.streams.config.StreamsConfigurator; import org.apache.streams.core.StreamsDatum; import org.apache.streams.elasticsearch.ElasticsearchClientManager; import org.apache.streams.elasticsearch.ElasticsearchReaderConfiguration; -import org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration; import org.apache.streams.elasticsearch.processor.DatumFromMetadataProcessor; import org.elasticsearch.client.Client; -import org.elasticsearch.test.ESIntegTestCase; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; import java.util.Map; -import java.util.Properties; -import java.util.concurrent.TimeUnit; /** * Created by sblackmon on 10/20/14. @@ -67,12 +58,7 @@ public class DatumFromMetadataProcessorIT { File conf_file = new File("target/test-classes/DatumFromMetadataProcessorIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - Properties es_properties = new Properties(); - InputStream es_stream = new FileInputStream("elasticsearch.properties"); - es_properties.load(es_stream); - Config esProps = ConfigFactory.parseProperties(es_properties); - Config typesafe = testResourceConfig.withFallback(esProps).withFallback(reference).resolve(); - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(ElasticsearchReaderConfiguration.class).detectConfiguration(typesafe, "elasticsearch"); testClient = new ElasticsearchClientManager(testConfiguration).getClient(); http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/095c082f/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildUpdaterIT.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildUpdaterIT.java b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildUpdaterIT.java index c37f920..6344028 100644 --- a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildUpdaterIT.java +++ b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildUpdaterIT.java @@ -96,12 +96,7 @@ public class ElasticsearchParentChildUpdaterIT { File conf_file = new File("target/test-classes/ElasticsearchParentChildUpdaterIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - Properties es_properties = new Properties(); - InputStream es_stream = new FileInputStream("elasticsearch.properties"); - es_properties.load(es_stream); - Config esProps = ConfigFactory.parseProperties(es_properties); - Config typesafe = testResourceConfig.withFallback(esProps).withFallback(reference).resolve(); - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(ElasticsearchWriterConfiguration.class).detectConfiguration(typesafe, "elasticsearch"); testClient = new ElasticsearchClientManager(testConfiguration).getClient(); http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/095c082f/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildWriterIT.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildWriterIT.java b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildWriterIT.java index 7254913..637fdfc 100644 --- a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildWriterIT.java +++ b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchParentChildWriterIT.java @@ -21,18 +21,14 @@ package org.apache.streams.elasticsearch.test; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Strings; -import com.google.common.collect.Lists; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import com.typesafe.config.ConfigParseOptions; import org.apache.commons.io.Charsets; import org.apache.commons.io.IOUtils; import org.apache.streams.config.ComponentConfigurator; -import org.apache.streams.config.StreamsConfiguration; -import org.apache.streams.config.StreamsConfigurator; import org.apache.streams.core.StreamsDatum; import org.apache.streams.elasticsearch.ElasticsearchClientManager; -import org.apache.streams.elasticsearch.ElasticsearchPersistUpdater; import org.apache.streams.elasticsearch.ElasticsearchPersistWriter; import org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration; import org.apache.streams.jackson.StreamsJacksonMapper; @@ -45,15 +41,11 @@ import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; -import org.elasticsearch.action.count.CountRequest; -import org.elasticsearch.action.count.CountResponse; -import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.index.query.QueryBuilders; import org.junit.Before; import org.junit.Test; import org.reflections.Reflections; @@ -64,16 +56,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileInputStream; import java.io.InputStream; import java.net.URL; import java.util.List; -import java.util.Properties; import java.util.Set; import static junit.framework.TestCase.assertTrue; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; /** @@ -99,12 +88,7 @@ public class ElasticsearchParentChildWriterIT { File conf_file = new File("target/test-classes/ElasticsearchParentChildWriterIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - Properties es_properties = new Properties(); - InputStream es_stream = new FileInputStream("elasticsearch.properties"); - es_properties.load(es_stream); - Config esProps = ConfigFactory.parseProperties(es_properties); - Config typesafe = testResourceConfig.withFallback(esProps).withFallback(reference).resolve(); - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(ElasticsearchWriterConfiguration.class).detectConfiguration(typesafe, "elasticsearch"); testClient = new ElasticsearchClientManager(testConfiguration).getClient(); http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/095c082f/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistUpdaterIT.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistUpdaterIT.java b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistUpdaterIT.java index ab6337f..56f102b 100644 --- a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistUpdaterIT.java +++ b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistUpdaterIT.java @@ -81,12 +81,7 @@ public class ElasticsearchPersistUpdaterIT { File conf_file = new File("target/test-classes/ElasticsearchPersistUpdaterIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - Properties es_properties = new Properties(); - InputStream es_stream = new FileInputStream("elasticsearch.properties"); - es_properties.load(es_stream); - Config esProps = ConfigFactory.parseProperties(es_properties); - Config typesafe = testResourceConfig.withFallback(esProps).withFallback(reference).resolve(); - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(ElasticsearchWriterConfiguration.class).detectConfiguration(typesafe, "elasticsearch"); testClient = new ElasticsearchClientManager(testConfiguration).getClient(); http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/095c082f/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistWriterIT.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistWriterIT.java b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistWriterIT.java index e4dfba2..f291dcd 100644 --- a/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistWriterIT.java +++ b/streams-contrib/streams-persist-elasticsearch/src/test/java/org/apache/streams/elasticsearch/test/ElasticsearchPersistWriterIT.java @@ -19,7 +19,6 @@ package org.apache.streams.elasticsearch.test; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import com.typesafe.config.ConfigParseOptions; @@ -29,14 +28,11 @@ import org.apache.streams.config.ComponentConfigurator; import org.apache.streams.config.StreamsConfiguration; import org.apache.streams.config.StreamsConfigurator; import org.apache.streams.core.StreamsDatum; -import org.apache.streams.elasticsearch.ElasticsearchClient; import org.apache.streams.elasticsearch.ElasticsearchClientManager; -import org.apache.streams.elasticsearch.ElasticsearchPersistUpdater; import org.apache.streams.elasticsearch.ElasticsearchPersistWriter; import org.apache.streams.elasticsearch.ElasticsearchWriterConfiguration; import org.apache.streams.jackson.StreamsJacksonMapper; import org.apache.streams.pojo.json.Activity; -import org.apache.streams.pojo.json.Actor; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; @@ -48,22 +44,17 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.index.query.QueryBuilders; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Suite; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileInputStream; import java.io.InputStream; -import java.util.*; +import java.util.List; import static junit.framework.TestCase.assertTrue; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; /** @@ -85,12 +76,7 @@ public class ElasticsearchPersistWriterIT { File conf_file = new File("target/test-classes/ElasticsearchPersistWriterIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - Properties es_properties = new Properties(); - InputStream es_stream = new FileInputStream("elasticsearch.properties"); - es_properties.load(es_stream); - Config esProps = ConfigFactory.parseProperties(es_properties); - Config typesafe = testResourceConfig.withFallback(esProps).withFallback(reference).resolve(); - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(ElasticsearchWriterConfiguration.class).detectConfiguration(typesafe, "elasticsearch"); testClient = new ElasticsearchClientManager(testConfiguration).getClient(); http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/095c082f/streams-contrib/streams-persist-mongo/pom.xml ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-persist-mongo/pom.xml b/streams-contrib/streams-persist-mongo/pom.xml index 7534283..15b6c69 100644 --- a/streams-contrib/streams-persist-mongo/pom.xml +++ b/streams-contrib/streams-persist-mongo/pom.xml @@ -210,7 +210,7 @@ <ports> <port>${mongo.tcp.host}:${mongo.tcp.port}:27017</port> </ports> - <portPropertyFile>${pom.basedir}/mongo.properties</portPropertyFile> + <portPropertyFile>mongo.properties</portPropertyFile> <log> <enabled>true</enabled> <date>default</date> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/095c082f/streams-contrib/streams-persist-mongo/src/test/java/org/apache/streams/mongo/test/MongoPersistIT.java ---------------------------------------------------------------------- diff --git a/streams-contrib/streams-persist-mongo/src/test/java/org/apache/streams/mongo/test/MongoPersistIT.java b/streams-contrib/streams-persist-mongo/src/test/java/org/apache/streams/mongo/test/MongoPersistIT.java index 6860b1a..18f5a62 100644 --- a/streams-contrib/streams-persist-mongo/src/test/java/org/apache/streams/mongo/test/MongoPersistIT.java +++ b/streams-contrib/streams-persist-mongo/src/test/java/org/apache/streams/mongo/test/MongoPersistIT.java @@ -19,15 +19,12 @@ package org.apache.streams.mongo.test; import com.fasterxml.jackson.databind.ObjectMapper; -import com.mongodb.MongoClient; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import com.typesafe.config.ConfigParseOptions; import org.apache.commons.io.Charsets; import org.apache.commons.io.IOUtils; import org.apache.streams.config.ComponentConfigurator; -import org.apache.streams.config.StreamsConfiguration; -import org.apache.streams.config.StreamsConfigurator; import org.apache.streams.core.StreamsDatum; import org.apache.streams.core.StreamsResultSet; import org.apache.streams.jackson.StreamsJacksonMapper; @@ -41,13 +38,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileInputStream; import java.io.InputStream; import java.util.List; -import java.util.Properties; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; /** * Test writing documents @@ -69,12 +63,7 @@ public class MongoPersistIT { File conf_file = new File("target/test-classes/MongoPersistIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); - Properties mongo_properties = new Properties(); - InputStream mongo_stream = new FileInputStream("mongo.properties"); - mongo_properties.load(mongo_stream); - Config mongoProps = ConfigFactory.parseProperties(mongo_properties); - Config typesafe = testResourceConfig.withFallback(mongoProps).withFallback(reference).resolve(); - StreamsConfiguration streams = StreamsConfigurator.detectConfiguration(typesafe); + Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(MongoConfiguration.class).detectConfiguration(typesafe, "mongo"); }