Repository: incubator-sdap-nexus Updated Branches: refs/heads/master 31f7ac2ee -> 2010cd289
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/be88efc5/nexus-ingest/nexus-sink/src/test/groovy/org/nasa/jpl/nexus/ingest/nexussink/NexusSinkIntegrationTest.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/nexus-sink/src/test/groovy/org/nasa/jpl/nexus/ingest/nexussink/NexusSinkIntegrationTest.groovy b/nexus-ingest/nexus-sink/src/test/groovy/org/nasa/jpl/nexus/ingest/nexussink/NexusSinkIntegrationTest.groovy index 96e788b..d806a3b 100644 --- a/nexus-ingest/nexus-sink/src/test/groovy/org/nasa/jpl/nexus/ingest/nexussink/NexusSinkIntegrationTest.groovy +++ b/nexus-ingest/nexus-sink/src/test/groovy/org/nasa/jpl/nexus/ingest/nexussink/NexusSinkIntegrationTest.groovy @@ -4,8 +4,10 @@ *****************************************************************************/ package org.nasa.jpl.nexus.ingest.nexussink +import com.amazonaws.regions.Region +import com.amazonaws.regions.Regions +import com.amazonaws.services.s3.AmazonS3Client import com.datastax.driver.core.Cluster -import io.findify.s3mock.S3Mock import org.apache.solr.client.solrj.SolrClient import org.apache.solr.common.params.ModifiableSolrParams import org.cassandraunit.utils.EmbeddedCassandraServerHelper @@ -41,7 +43,6 @@ public class NexusSinkIntegrationTest { private static SingleNodeApplication application - private static final String CASSANDRA_CONFIG = "spring-cassandra.yaml" private static final String CASSANDRA_KEYSPACE = "testNexusSink" @@ -50,9 +51,14 @@ public class NexusSinkIntegrationTest { private static final String CONTACT = "127.0.0.1" + private static final String S3_BUCKET_NAME = "nexus-jpl" + private static final String AWS_REGION = "us-west-2" + private static final String DYNAMO_TABLE_NAME = "nexus-jpl-table" + private static Cluster cluster private static CassandraOperations cassandraTemplate + private static AmazonS3Client s3Client private static String SOLR_URL = "http://embedded"; private static String SOLR_CORE = "nexustiles" @@ -62,19 +68,14 @@ public class NexusSinkIntegrationTest { @BeforeClass public static void setUpXd() { - S3Mock api = new S3Mock.Builder().withPort(8001).withInMemoryBackend().build(); - initCassandra() + initS3() - new RandomConfigurationSupport() - - RandomConfigurationSupport randomConfigSupport = new RandomConfigurationSupport(); + RandomConfigurationSupport randomConfigSupport = new RandomConfigurationSupport() application = new SingleNodeApplication().run() - singleNodeIntegrationTestSupport = new SingleNodeIntegrationTestSupport - (application); - singleNodeIntegrationTestSupport.addModuleRegistry(new SingletonModuleRegistry(ModuleType.sink, - MODULE_NAME)); + singleNodeIntegrationTestSupport = new SingleNodeIntegrationTestSupport(application) + singleNodeIntegrationTestSupport.addModuleRegistry(new SingletonModuleRegistry(ModuleType.sink, MODULE_NAME)) } @@ -99,6 +100,11 @@ public class NexusSinkIntegrationTest { } } + private static void initS3() { + s3Client = new AmazonS3Client() + s3Client.setRegion(Region.getRegion(Regions.fromName(AWS_REGION))) + } + @AfterClass public static void cleanup() { @@ -122,6 +128,80 @@ public class NexusSinkIntegrationTest { } } + @Test + public void testS3() { + + def streamName = "testNexusSink" + + NexusContent.NexusTile tile = NexusContent.NexusTile.newBuilder() + .setTile(NexusContent.TileData.newBuilder() + .setTileId(UUID.randomUUID().toString()) + .setGridTile( + NexusContent.GridTile.newBuilder() + .build()) + .build()) + .setSummary(NexusContent.TileSummary.newBuilder() + .setTileId("1") + .setBbox(NexusContent.TileSummary.BBox.newBuilder() + .setLatMin(51) + .setLatMax(55) + .setLonMin(22) + .setLonMax(30) + .build()) + .setDatasetName("test") + .setDatasetUuid("4") + .setDataVarName("sst") + .setGranule("test.nc") + .setSectionSpec("0:1,0:1") + .setStats(NexusContent.TileSummary.DataStats.newBuilder() + .setCount(10) + .setMax(50) + .setMin(50) + .setMean(50) + .setMaxTime(500000) + .setMinTime(500000) + .build()) + .addGlobalAttributes(NexusContent.Attribute.newBuilder() + .setName("day_of_year_i") + .addValues("006") + .build()) + .addGlobalAttributes(NexusContent.Attribute.newBuilder() + .setName("attr_multi_value_test") + .addValues("006") + .addValues("multi") + .build()) + .build()) + .build() + + def processingChainUnderTest = "$MODULE_NAME --$PROPERTY_NAME_SOLR_SERVER_URL=$SOLR_URL --$PROPERTY_NAME_SOLR_COLLECTION=$SOLR_CORE --$PROPERTY_NAME_S3_BUCKET=$S3_BUCKET_NAME --$PROPERTY_NAME_AWS_REGION=$AWS_REGION" + + SingleNodeProcessingChainProducer chain = chainProducer(application, streamName, processingChainUnderTest) + SolrClient solrClient = singleNodeIntegrationTestSupport.getModule(streamName, MODULE_NAME, 0).applicationContext.getBean("solrClient", SolrClient) + + chain.sendPayload(tile) + + assertEqualsEventually 1, new Supplier<Integer>() { + @Override + Integer get() { + solrClient.commit() + def q = solrClient.query(new ModifiableSolrParams().add("q", "*:*")) + println "query: $q" + return q.results.numFound + } + } + + /* + assertEqualsEventually 1, new Supplier<Integer>() { + @Override + Integer get() { + ObjectListing objectListing = s3Client.listObjects(new ListObjectsRequest().withBucketName(S3_BUCKET_NAME)) + return objectListing.getObjectSummaries().size() + } + }*/ + + chain.destroy() + } + /** * Test a Nexus sink module */ @@ -651,12 +731,42 @@ public class NexusSinkIntegrationTest { } @Test(expected = ModuleConfigurationException.class) - public void testSolrZkAndSolrURLMutalExclusive() throws Exception { + public void testCassandraAndS3MutualExclusive() throws Exception { - def streamName = "testSolrZkAndSolrURLMutalExclusive" + def streamName = "testCassandraAndS3MutualExclusive" + def processingChainUnderTest = "$MODULE_NAME --$PROPERTY_NAME_CASSANDRA_CONTACT_POINTS=$CONTACT --$PROPERTY_NAME_CASSANDRA_KEYSPACE=$CASSANDRA_KEYSPACE --$PROPERTY_NAME_CASSANDRA_PORT=$PORT --$PROPERTY_NAME_SOLR_SERVER_URL=$SOLR_URL --$PROPERTY_NAME_SOLR_COLLECTION=$SOLR_CORE --$PROPERTY_NAME_S3_BUCKET=$S3_BUCKET_NAME" + assertNull(chainProducer(application, streamName, processingChainUnderTest)) + } - def processingChainUnderTest = "$MODULE_NAME --$PROPERTY_NAME_SOLR_SERVER_URL=$SOLR_URL --$PROPERTY_NAME_SOLR_CLOUD_ZK_URL=zk1 --$PROPERTY_NAME_CASSANDRA_CONTACT_POINTS=$CONTACT --$PROPERTY_NAME_CASSANDRA_KEYSPACE=$CASSANDRA_KEYSPACE --$PROPERTY_NAME_CASSANDRA_PORT=$PORT --$PROPERTY_NAME_SOLR_COLLECTION=$SOLR_CORE" + @Test(expected = ModuleConfigurationException.class) + public void testCassandraConfigured() throws Exception { + + def streamName = "testCassandraConfigured" + def processingChainUnderTest = "$MODULE_NAME --$PROPERTY_NAME_CASSANDRA_KEYSPACE=$CASSANDRA_KEYSPACE --$PROPERTY_NAME_CASSANDRA_PORT=$PORT --$PROPERTY_NAME_SOLR_SERVER_URL=$SOLR_URL --$PROPERTY_NAME_SOLR_COLLECTION=$SOLR_CORE" + assertNull(chainProducer(application, streamName, processingChainUnderTest)) + } + + @Test(expected = ModuleConfigurationException.class) + public void testS3Configured() throws Exception { + def streamName = "testS3Configured" + def processingChainUnderTest = "$MODULE_NAME --$PROPERTY_NAME_SOLR_SERVER_URL=$SOLR_URL --$PROPERTY_NAME_SOLR_COLLECTION=$SOLR_CORE --$PROPERTY_NAME_S3_BUCKET=$S3_BUCKET_NAME" + assertNull(chainProducer(application, streamName, processingChainUnderTest)) + } + + @Test(expected = ModuleConfigurationException.class) + public void testDynamoConfigured() throws Exception { + + def streamName = "testDynamoConfigured" + def processingChainUnderTest = "$MODULE_NAME --$PROPERTY_NAME_SOLR_SERVER_URL=$SOLR_URL --$PROPERTY_NAME_SOLR_COLLECTION=$SOLR_CORE --$PROPERTY_NAME_DYNAMO_TABLE_NAME=$DYNAMO_TABLE_NAME" + assertNull(chainProducer(application, streamName, processingChainUnderTest)) + } + + @Test(expected = ModuleConfigurationException.class) + public void testSolrZkAndSolrURLMutualExclusive() throws Exception { + + def streamName = "testSolrZkAndSolrURLMutualExclusive" + def processingChainUnderTest = "$MODULE_NAME --$PROPERTY_NAME_SOLR_SERVER_URL=$SOLR_URL --$PROPERTY_NAME_SOLR_CLOUD_ZK_URL=zk1 --$PROPERTY_NAME_CASSANDRA_CONTACT_POINTS=$CONTACT --$PROPERTY_NAME_CASSANDRA_KEYSPACE=$CASSANDRA_KEYSPACE --$PROPERTY_NAME_CASSANDRA_PORT=$PORT --$PROPERTY_NAME_SOLR_COLLECTION=$SOLR_CORE" assertNull(chainProducer(application, streamName, processingChainUnderTest)) } http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/be88efc5/nexus-ingest/nexus-sink/src/test/groovy/org/nasa/jpl/nexus/ingest/nexussink/SolrStoreUnitTest.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/nexus-sink/src/test/groovy/org/nasa/jpl/nexus/ingest/nexussink/SolrStoreUnitTest.groovy b/nexus-ingest/nexus-sink/src/test/groovy/org/nasa/jpl/nexus/ingest/nexussink/SolrStoreUnitTest.groovy new file mode 100644 index 0000000..5b69b26 --- /dev/null +++ b/nexus-ingest/nexus-sink/src/test/groovy/org/nasa/jpl/nexus/ingest/nexussink/SolrStoreUnitTest.groovy @@ -0,0 +1,68 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ +package org.nasa.jpl.nexus.ingest.nexussink + +import org.junit.Test +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent +import org.springframework.mock.env.MockEnvironment + +import static org.junit.Assert.assertEquals + +/** + * Created by greguska on 5/2/17. + */ +class SolrStoreUnitTest { + + @Test + public void testGetSolrDocFromTileSummary() { + def solrStore = new SolrStore(null); + solrStore.setEnvironment(new MockEnvironment()) + + def tileSummary = NexusContent.TileSummary.newBuilder() + .setTileId("1") + .setBbox(NexusContent.TileSummary.BBox.newBuilder() + .setLatMin(51) + .setLatMax(55) + .setLonMin(22) + .setLonMax(30) + .build()) + .setDatasetName("test") + .setDatasetUuid("4") + .setDataVarName("sst") + .setGranule("test.nc") + .setSectionSpec("0:1,0:1") + .setStats(NexusContent.TileSummary.DataStats.newBuilder() + .setCount(10) + .setMax(50) + .setMin(50) + .setMean(50) + .setMaxTime(1429142399) + .setMinTime(1429142399) + .build()) + .build() + + def doc = solrStore.getSolrDocFromTileSummary(tileSummary) + + assertEquals("2015-04-15T23:59:59Z", doc.get("tile_min_time_dt").value) + assertEquals("2015-04-15T23:59:59Z", doc.get("tile_max_time_dt").value) + assertEquals("sea_surface_temp", doc.get('table_s').value) + assertEquals("POLYGON((22.0 51.0, 30.0 51.0, 30.0 55.0, 22.0 55.0, 22.0 51.0))", doc.get('geo').value) + assertEquals("1", doc.get('id').value) + assertEquals("4", doc.get('dataset_id_s').value) + assertEquals("0:1,0:1", doc.get('sectionSpec_s').value) + assertEquals("test", doc.get('dataset_s').value) + assertEquals("test.nc", doc.get('granule_s').value) + assertEquals("sst", doc.get('tile_var_name_s').value) + assertEquals(22.0f, (Float) doc.get('tile_min_lon').value, 0.01f) + assertEquals(30.0f, (Float) doc.get('tile_max_lon').value, 0.01f) + assertEquals(51.0f, (Float) doc.get('tile_min_lat').value, 0.01f) + assertEquals(55.0f, (Float) doc.get('tile_max_lat').value, 0.01f) + assertEquals(50.0f, (Float) doc.get('tile_min_val_d').value, 0.01f) + assertEquals(50.0f, (Float) doc.get('tile_max_val_d').value, 0.01f) + assertEquals(50.0f, (Float) doc.get('tile_avg_val_d').value, 0.01f) + assertEquals(10, doc.get('tile_count_i').value) + assertEquals("test!1", doc.get('solr_id_s').value) + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/be88efc5/nexus-ingest/nexus-sink/src/test/java/org/nasa/jpl/nexus/ingest/nexussink/NexusSinkOptionsIntegrationTest.java ---------------------------------------------------------------------- diff --git a/nexus-ingest/nexus-sink/src/test/java/org/nasa/jpl/nexus/ingest/nexussink/NexusSinkOptionsIntegrationTest.java b/nexus-ingest/nexus-sink/src/test/java/org/nasa/jpl/nexus/ingest/nexussink/NexusSinkOptionsIntegrationTest.java index 374483d..7fa1824 100644 --- a/nexus-ingest/nexus-sink/src/test/java/org/nasa/jpl/nexus/ingest/nexussink/NexusSinkOptionsIntegrationTest.java +++ b/nexus-ingest/nexus-sink/src/test/java/org/nasa/jpl/nexus/ingest/nexussink/NexusSinkOptionsIntegrationTest.java @@ -13,9 +13,7 @@ import org.springframework.xd.module.options.ModuleOption; import org.springframework.xd.module.options.ModuleOptionsMetadata; import org.springframework.xd.module.options.ModuleOptionsMetadataResolver; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasProperty; +import static org.hamcrest.Matchers.*; import static org.junit.Assert.assertThat; import static org.nasa.jpl.nexus.ingest.nexussink.NexusSinkOptionsMetadata.*; import static org.springframework.xd.module.ModuleType.processor; @@ -41,7 +39,10 @@ public class NexusSinkOptionsIntegrationTest { moduleOptionNamed(PROPERTY_NAME_CASSANDRA_CONTACT_POINTS), moduleOptionNamed(PROPERTY_NAME_CASSANDRA_KEYSPACE), moduleOptionNamed(PROPERTY_NAME_CASSANDRA_PORT), - moduleOptionNamed(PROPERTY_NAME_INSERT_BUFFER))); + moduleOptionNamed(PROPERTY_NAME_INSERT_BUFFER), + moduleOptionNamed(PROPERTY_NAME_S3_BUCKET), + moduleOptionNamed(PROPERTY_NAME_AWS_REGION), + moduleOptionNamed(PROPERTY_NAME_DYNAMO_TABLE_NAME))); } public static Matcher<ModuleOption> moduleOptionNamed(String name) { http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/be88efc5/nexus-ingest/nexus-sink/src/test/java/org/nasa/jpl/nexus/ingest/nexussink/TestInfrastructureConfiguration.java ---------------------------------------------------------------------- diff --git a/nexus-ingest/nexus-sink/src/test/java/org/nasa/jpl/nexus/ingest/nexussink/TestInfrastructureConfiguration.java b/nexus-ingest/nexus-sink/src/test/java/org/nasa/jpl/nexus/ingest/nexussink/TestInfrastructureConfiguration.java index 84f85ec..17c607a 100644 --- a/nexus-ingest/nexus-sink/src/test/java/org/nasa/jpl/nexus/ingest/nexussink/TestInfrastructureConfiguration.java +++ b/nexus-ingest/nexus-sink/src/test/java/org/nasa/jpl/nexus/ingest/nexussink/TestInfrastructureConfiguration.java @@ -4,6 +4,8 @@ *****************************************************************************/ package org.nasa.jpl.nexus.ingest.nexussink; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.S3ClientOptions; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; import org.springframework.beans.factory.annotation.Value; @@ -47,6 +49,35 @@ public class TestInfrastructureConfiguration { public SolrOperations solrTemplate(SolrClient solrClient) { return new SolrTemplate(solrClient); } + + @Bean + public MetadataStore metadataStore(SolrOperations solrTemplate) { + MetadataStore metadataStore = new SolrStore(solrTemplate); + return metadataStore; + } + + } + + @Configuration + @Profile("s3local") + static class S3LocalConfiguration { + @Value("#{environment[T(org.nasa.jpl.nexus.ingest.nexussink.NexusSinkOptionsMetadata).PROPERTY_NAME_S3_BUCKET]}") + private String s3BucketName; + + @Bean + public AmazonS3Client s3client() { + AmazonS3Client s3Client = new AmazonS3Client(); + S3ClientOptions s3ClientOptions = S3ClientOptions.builder().setPathStyleAccess(true).build(); + s3Client.setS3ClientOptions(s3ClientOptions); + s3Client.setEndpoint("http://localhost:8080";); + return s3Client; + } + + @Bean + public DataStore dataStore(AmazonS3Client s3Client) { + S3Store s3Store = new S3Store(s3Client, s3BucketName); + return s3Store; + } } }
