got integration tests working
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/787b59fa Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/787b59fa Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/787b59fa Branch: refs/heads/master Commit: 787b59fa54d27ff2ed4d780889a9394131fd97df Parents: 9a51dbe Author: Frank Greguska <[email protected]> Authored: Tue Jan 9 16:09:50 2018 -0800 Committer: Frank Greguska <[email protected]> Committed: Tue Jan 9 16:09:50 2018 -0800 ---------------------------------------------------------------------- build.gradle | 6 ++ settings.gradle | 2 +- .../ningester/configuration/AppConfig.java | 27 +++++ .../ningester/configuration/BatchConfig.java | 12 +-- .../configuration/DatasourceConfig.java | 107 ++++++------------- .../properties/DatasourceProperties.java | 11 +- .../processors/PythonChainProcessor.java | 4 +- .../nexus/ningester/writer/CassandraStore.java | 6 +- .../nexus/ningester/writer/MetadataStore.java | 2 + .../jpl/nexus/ningester/writer/NexusWriter.java | 45 ++++++++ .../jpl/nexus/ningester/writer/SolrStore.java | 83 ++++++++------ .../writer/properties/CassandraStore.java | 33 ++---- .../ningester/writer/properties/SolrStore.java | 27 ++--- src/main/resources/application.yml | 6 +- .../datatiler/NetCDFItemReaderTest.java | 2 - .../nexus/ningester/writer/SolrStoreTest.java | 2 - .../nexus/ningester/testjobs/AvhrrJobTest.java | 56 ++++++++-- .../nexus/ningester/testjobs/SmapJobTest.java | 57 ++++++++-- .../jpl/nexus/ningester/testjobs/TestUtils.java | 28 +++++ .../resources/testjobs/AvhrrJobTest.yml | 34 +++--- src/testJobs/resources/testjobs/SmapJobTest.yml | 100 +++++++++-------- 21 files changed, 405 insertions(+), 245 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index a7e6c6d..ff30ed2 100644 --- a/build.gradle +++ b/build.gradle @@ -7,6 +7,10 @@ plugins { id 'java' id 'idea' + id 'propdeps' + id 'propdeps-maven' + id 'propdeps-idea' + id 'propdeps-eclipse' id 'org.springframework.boot' version '1.5.9.RELEASE' id 'org.unbroken-dome.test-sets' version '1.4.2' } @@ -41,6 +45,8 @@ dependencyManagement { } dependencies { + optional "org.springframework.boot:spring-boot-configuration-processor" + compile("org.springframework.boot:spring-boot-starter-batch") compile("org.springframework:spring-web") compile("org.springframework.data:spring-data-cassandra") http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/settings.gradle ---------------------------------------------------------------------- diff --git a/settings.gradle b/settings.gradle index 82729b6..7acaab2 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,7 +1,7 @@ pluginManagement { resolutionStrategy { eachPlugin { - if (requested.id.namespace?.startsWith('propdeps')) { + if (requested.id.name?.startsWith('propdeps')) { useModule('io.spring.gradle:propdeps-plugin:0.0.9.RELEASE') } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java index 3354c40..f30b627 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java @@ -11,6 +11,9 @@ import gov.nasa.jpl.nexus.ningester.datatiler.SliceFileByDimension; import gov.nasa.jpl.nexus.ningester.datatiler.SliceFileByTilesDesired; import gov.nasa.jpl.nexus.ningester.http.NexusTileConverter; import gov.nasa.jpl.nexus.ningester.processors.*; +import gov.nasa.jpl.nexus.ningester.writer.DataStore; +import gov.nasa.jpl.nexus.ningester.writer.MetadataStore; +import gov.nasa.jpl.nexus.ningester.writer.NexusWriter; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.batch.core.configuration.annotation.JobScope; import org.springframework.batch.item.ItemProcessor; @@ -86,6 +89,30 @@ public class AppConfig { return template; } + @Bean + public MetadataStore metadataStore() { + return new MetadataStore() { + @Override + public void saveMetadata(List<? extends NexusContent.NexusTile> nexusTiles) { + } + + @Override + public void deleteMetadata(List<? extends NexusContent.NexusTile> nexusTiles) { + } + }; + } + + @Bean + public DataStore dataStore() { + return nexusTiles -> { + }; + } + + @Bean + public NexusWriter nexusWriter(MetadataStore metadataStore, DataStore dataStore) { + return new NexusWriter(metadataStore, dataStore); + } + /* * Item Processor beans defined below */ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java index d333dc7..5bd108f 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java @@ -6,6 +6,7 @@ import gov.nasa.jpl.nexus.ningester.datatiler.NetCDFItemReader; import gov.nasa.jpl.nexus.ningester.processors.CompositeItemProcessor; import gov.nasa.jpl.nexus.ningester.writer.DataStore; import gov.nasa.jpl.nexus.ningester.writer.MetadataStore; +import gov.nasa.jpl.nexus.ningester.writer.NexusWriter; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; @@ -73,15 +74,8 @@ public class BatchConfig { @Bean @JobScope - protected ItemWriter<NexusContent.NexusTile> writer(DataStore dataStore, MetadataStore metadataStore) { - - CompositeItemWriter<NexusContent.NexusTile> compositeItemWriter = new CompositeItemWriter<>(); - compositeItemWriter.setDelegates(Arrays.asList( - metadataStore::saveMetadata, - dataStore::saveData) - ); - - return compositeItemWriter; + protected ItemWriter<NexusContent.NexusTile> writer(NexusWriter nexusWriter) { + return nexusWriter::saveToNexus; } @Bean http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java index de4fef2..8d52f59 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java @@ -5,17 +5,23 @@ import com.amazonaws.regions.Regions; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; import com.amazonaws.services.s3.AmazonS3Client; +import com.datastax.driver.core.ProtocolVersion; import gov.nasa.jpl.nexus.ningester.configuration.properties.DatasourceProperties; import gov.nasa.jpl.nexus.ningester.writer.*; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration; +import org.springframework.boot.autoconfigure.cassandra.ClusterBuilderCustomizer; +import org.springframework.boot.autoconfigure.data.cassandra.CassandraDataAutoConfiguration; +import org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration; +import org.springframework.boot.autoconfigure.solr.SolrProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Profile; +import org.springframework.core.env.Environment; import org.springframework.data.cassandra.config.CassandraClusterFactoryBean; import org.springframework.data.cassandra.config.CassandraSessionFactoryBean; import org.springframework.data.cassandra.config.SchemaAction; @@ -31,17 +37,12 @@ import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; import javax.sql.DataSource; +import java.util.Collections; +import java.util.List; @Configuration public class DatasourceConfig { - private final DatasourceProperties datasourceProperties; - - @Autowired - public DatasourceConfig(DatasourceProperties datasourceProperties) { - this.datasourceProperties = datasourceProperties; - } - @Bean @Profile("embedded") public DataSource dataSource() { @@ -54,53 +55,22 @@ public class DatasourceConfig { @Configuration @Profile("cassandra") - class CassandraConfiguration { - @Bean - public CassandraClusterFactoryBean cluster() { - - CassandraClusterFactoryBean cluster = new CassandraClusterFactoryBean(); - cluster.setContactPoints(datasourceProperties.getCassandraStore().getContactPoints()); - cluster.setPort(datasourceProperties.getCassandraStore().getPort()); - - return cluster; - } - - @Bean - public CassandraMappingContext mappingContext() { - return new BasicCassandraMappingContext(); - } - - @Bean - public CassandraConverter converter() { - return new MappingCassandraConverter(mappingContext()); - } - - @Bean - public CassandraSessionFactoryBean session(){ - - CassandraSessionFactoryBean session = new CassandraSessionFactoryBean(); - session.setCluster(cluster().getObject()); - session.setKeyspaceName(datasourceProperties.getCassandraStore().getKeyspace()); - session.setConverter(converter()); - session.setSchemaAction(SchemaAction.NONE); - - return session; - } - - @Bean - public CassandraOperations cassandraTemplate(){ - return new CassandraTemplate(session().getObject()); - } + @Import({CassandraDataAutoConfiguration.class, CassandraAutoConfiguration.class}) + static class CassandraConfiguration { @Bean - public DataStore dataStore(CassandraOperations cassandraTemplate) { + public DataStore dataStore(CassandraTemplate cassandraTemplate) { return new CassandraStore(cassandraTemplate); } } @Configuration @Profile("dynamo") - class DynamoConfiguration { + static class DynamoConfiguration { + + @Autowired + private DatasourceProperties datasourceProperties; + @Bean public AmazonDynamoDB dynamoClient() { AmazonDynamoDB dynamoClient = new AmazonDynamoDBClient(); @@ -118,7 +88,10 @@ public class DatasourceConfig { @Configuration @Profile("s3") - class S3Configuration { + static class S3Configuration { + @Autowired + private DatasourceProperties datasourceProperties; + @Bean public AmazonS3Client s3client() { AmazonS3Client s3Client = new AmazonS3Client(); @@ -133,41 +106,27 @@ public class DatasourceConfig { } @Configuration - @Profile("solr-standalone") - class SolrConfiguration { - @Bean - public SolrClient solrClient(){ return new HttpSolrClient(datasourceProperties.getSolrStore().getUrl() + datasourceProperties.getSolrStore().getCollection());} - - @Bean - public SolrOperations solrTemplate(SolrClient solrClient) { - return new SolrTemplate(solrClient); - } - - @Bean - public MetadataStore metadataStore(SolrOperations solrTemplate) { - return new SolrStore(solrTemplate); - } - } - - @Configuration - @Profile("solr-cloud") - class SolrCloudConfiguration { - @Bean - public SolrClient solrClient(){ - CloudSolrClient client = new CloudSolrClient(datasourceProperties.getSolrStore().getZkHost()); - client.setDefaultCollection(datasourceProperties.getSolrStore().getCollection()); + @Profile("solr") + @Import({SolrAutoConfiguration.class}) + static class SolrConfiguration { - return client; - } + @Autowired + private DatasourceProperties datasourceProperties; @Bean public SolrOperations solrTemplate(SolrClient solrClient) { return new SolrTemplate(solrClient); } + @Bean public MetadataStore metadataStore(SolrOperations solrTemplate) { - return new SolrStore(solrTemplate); + SolrStore solrStore = new SolrStore(solrTemplate); + solrStore.setCollection(datasourceProperties.getSolrStore().getCollection()); + solrStore.setCommitWithin(datasourceProperties.getSolrStore().getCommitWithin()); + solrStore.setGeoPrecision(datasourceProperties.getSolrStore().getGeoPrecision()); + + return solrStore; } } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java index 228ec1e..9f5c5e7 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java @@ -1,4 +1,5 @@ -/***************************************************************************** +/* + ***************************************************************************** * Copyright (c) 2018 Jet Propulsion Laboratory, * California Institute of Technology. All rights reserved *****************************************************************************/ @@ -29,10 +30,6 @@ public class DatasourceProperties { @NestedConfigurationProperty private final SolrStore solrStore = new SolrStore(); - public CassandraStore getCassandraStore() { - return cassandraStore; - } - public DynamoStore getDynamoStore() { return dynamoStore; } @@ -44,4 +41,8 @@ public class DatasourceProperties { public SolrStore getSolrStore() { return solrStore; } + + public CassandraStore getCassandraStore() { + return cassandraStore; + } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java index 0488562..be6ec78 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java @@ -57,11 +57,13 @@ public class PythonChainProcessor { HttpEntity<PythonChainProcessorRequest> requestEntity = new HttpEntity<>(chainProcessorRequest, headers); - return restTemplate.exchange( + NexusContent.NexusTile outNexusTile = restTemplate.exchange( uriPath, HttpMethod.POST, requestEntity, NexusContent.NexusTile.class).getBody(); + + return outNexusTile; } public void setProcessorList(List<PythonProcessorModule> processorList) { http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java index 492d2cf..d3522a1 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java @@ -30,7 +30,7 @@ public class CassandraStore implements DataStore { @Override public void saveData(List<? extends NexusContent.NexusTile> nexusTiles) { - String query = "insert into ${tableName} (tile_id, tile_blob) VALUES (?, ?)"; + String query = "insert into "+ this.tableName + " (tile_id, tile_blob) VALUES (?, ?)"; cassandraTemplate.ingest(query, nexusTiles.stream() .map(nexusTile -> getCassandraRowFromTileData(nexusTile.getTile())) .collect(Collectors.toList())); @@ -41,4 +41,8 @@ public class CassandraStore implements DataStore { UUID tileId = UUID.fromString(tile.getTileId()); return Arrays.asList(tileId, ByteBuffer.wrap(tile.toByteArray())); } + + public void setTableName(String tableName) { + this.tableName = tableName; + } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/main/java/gov/nasa/jpl/nexus/ningester/writer/MetadataStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/MetadataStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/MetadataStore.java index 3baf6d5..c9323c5 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/MetadataStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/MetadataStore.java @@ -12,4 +12,6 @@ import java.util.List; public interface MetadataStore { void saveMetadata(List<? extends NexusContent.NexusTile> nexusTiles); + + void deleteMetadata(List<? extends NexusContent.NexusTile> nexusTiles); } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java new file mode 100644 index 0000000..63a98e1 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java @@ -0,0 +1,45 @@ +/* + **************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.writer; + +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class NexusWriter { + + private MetadataStore metadataStore; + private DataStore dataStore; + + private static final Logger log = LoggerFactory.getLogger(NexusWriter.class); + + public NexusWriter(MetadataStore metadataStore, DataStore dataStore) { + this.metadataStore = metadataStore; + this.dataStore = dataStore; + } + + public void saveToNexus(List<? extends NexusContent.NexusTile> nexusTiles) { + if (nexusTiles.size() > 0) { + metadataStore.saveMetadata(nexusTiles); + + try { + dataStore.saveData(nexusTiles); + + } catch (RuntimeException e) { + try { + metadataStore.deleteMetadata(nexusTiles); + } catch (RuntimeException e2) { + log.error("During exception while saving data, could not rollback metadata", e2); + } + throw e; + } + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java index 98a22fa..ed91ff5 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java @@ -21,13 +21,17 @@ import java.util.*; import java.util.stream.Collectors; public class SolrStore implements MetadataStore { - private Environment environment; + + private Integer commitWithin = 1000; + private Integer geoPrecision = 3; + private String collection = "nexustiles"; + private SolrOperations solr; private Logger log = LoggerFactory.getLogger(SolrStore.class); //TODO This will be refactored at some point to be dynamic per-message. Or maybe per-group. - private String tableName = "sea_surface_temp"; + private static final String TABLE_NAME = "sea_surface_temp"; private static final SimpleDateFormat iso = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); @@ -39,18 +43,22 @@ public class SolrStore implements MetadataStore { this.solr = solr; } - @Resource - public void setEnvironment(Environment environment) { - this.environment = environment; - } - @Override public void saveMetadata(List<? extends NexusContent.NexusTile> nexusTiles) { List<SolrInputDocument> solrdocs = nexusTiles.stream() .map(nexusTile -> getSolrDocFromTileSummary(nexusTile.getSummary())) .collect(Collectors.toList()); - solr.saveDocuments(solrdocs, environment.getProperty("solrCommitWithin", Integer.class, 1000)); + solr.saveDocuments(this.collection, solrdocs, commitWithin); + } + + @Override + public void deleteMetadata(List<? extends NexusContent.NexusTile> nexusTiles) { + + List<String> tileIds = nexusTiles.stream() + .map(nexusTile -> nexusTile.getSummary().getDatasetName() + "!" + nexusTile.getSummary().getTileId()) + .collect(Collectors.toList()); + solr.deleteById(this.collection, tileIds); } public SolrInputDocument getSolrDocFromTileSummary(NexusContent.TileSummary summary) { @@ -70,38 +78,37 @@ public class SolrStore implements MetadataStore { String granuleFileName = Paths.get(summary.getGranule()).getFileName().toString(); - Map<String, Object> doc = new HashMap<>(); - doc.put("table_s", tableName); - doc.put("geo", geo); - doc.put("id", summary.getTileId()); - doc.put("solr_id_s", summary.getDatasetName() + "!" + summary.getTileId()); - doc.put("dataset_id_s", summary.getDatasetUuid()); - doc.put("sectionSpec_s", summary.getSectionSpec()); - doc.put("dataset_s", summary.getDatasetName()); - doc.put("granule_s", granuleFileName); - doc.put("tile_var_name_s", summary.getDataVarName()); - doc.put("tile_min_lon", bbox.getLonMin()); - doc.put("tile_max_lon", bbox.getLonMax()); - doc.put("tile_min_lat", bbox.getLatMin()); - doc.put("tile_max_lat", bbox.getLatMax()); - doc.put("tile_min_time_dt", minTime); - doc.put("tile_max_time_dt", maxTime); - doc.put("tile_min_val_d", stats.getMin()); - doc.put("tile_max_val_d", stats.getMax()); - doc.put("tile_avg_val_d", stats.getMean()); - doc.put("tile_count_i", Long.valueOf(stats.getCount()).intValue()); + SolrInputDocument inputDocument = new SolrInputDocument(); + inputDocument.addField("table_s", TABLE_NAME); + inputDocument.addField("geo", geo); + inputDocument.addField("id", summary.getTileId()); + inputDocument.addField("solr_id_s", summary.getDatasetName() + "!" + summary.getTileId()); + inputDocument.addField("dataset_id_s", summary.getDatasetUuid()); + inputDocument.addField("sectionSpec_s", summary.getSectionSpec()); + inputDocument.addField("dataset_s", summary.getDatasetName()); + inputDocument.addField("granule_s", granuleFileName); + inputDocument.addField("tile_var_name_s", summary.getDataVarName()); + inputDocument.addField("tile_min_lon", bbox.getLonMin()); + inputDocument.addField("tile_max_lon", bbox.getLonMax()); + inputDocument.addField("tile_min_lat", bbox.getLatMin()); + inputDocument.addField("tile_max_lat", bbox.getLatMax()); + inputDocument.addField("tile_min_time_dt", minTime); + inputDocument.addField("tile_max_time_dt", maxTime); + inputDocument.addField("tile_min_val_d", stats.getMin()); + inputDocument.addField("tile_max_val_d", stats.getMax()); + inputDocument.addField("tile_avg_val_d", stats.getMean()); + inputDocument.addField("tile_count_i", Long.valueOf(stats.getCount()).intValue()); summary.getGlobalAttributesList().forEach(attribute -> - doc.put(attribute.getName(), attribute.getValuesCount() == 1 ? attribute.getValues(0) : attribute.getValuesList()) + inputDocument.addField(attribute.getName(), attribute.getValuesCount() == 1 ? attribute.getValues(0) : attribute.getValuesList()) ); - - return toSolrInputDocument(doc); + return inputDocument; } private String determineGeo(NexusContent.TileSummary summary) { //Solr cannot index a POLYGON where all corners are the same point or when there are only 2 distinct points (line). //Solr is configured for a specific precision so we need to round to that precision before checking equality. - Integer geoPrecision = environment.getProperty("solrGeoPrecision", Integer.class, 3); + Integer geoPrecision = this.geoPrecision; BigDecimal latMin = BigDecimal.valueOf(summary.getBbox().getLatMin()).setScale(geoPrecision, BigDecimal.ROUND_HALF_UP); BigDecimal latMax = BigDecimal.valueOf(summary.getBbox().getLatMax()).setScale(geoPrecision, BigDecimal.ROUND_HALF_UP); @@ -148,4 +155,16 @@ public class SolrStore implements MetadataStore { return solrDoc; } + + public void setCollection(String collection) { + this.collection = collection; + } + + public void setCommitWithin(Integer commitWithin) { + this.commitWithin = commitWithin; + } + + public void setGeoPrecision(Integer geoPrecision) { + this.geoPrecision = geoPrecision; + } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java index c6da68e..a2d40d5 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java @@ -1,4 +1,5 @@ -/***************************************************************************** +/* + **************************************************************************** * Copyright (c) 2018 Jet Propulsion Laboratory, * California Institute of Technology. All rights reserved *****************************************************************************/ @@ -8,35 +9,17 @@ package gov.nasa.jpl.nexus.ningester.writer.properties; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; -@ConfigurationProperties("cassandra") +@ConfigurationProperties @Component("cassandraStoreProperties") public class CassandraStore { - private String contactPoints; - private String keyspace; - private Integer port; + private String tableName = "sea_surface_temp"; - public String getContactPoints() { - return contactPoints; + public String getTableName() { + return tableName; } - public void setContactPoints(String contactPoints) { - this.contactPoints = contactPoints; - } - - public String getKeyspace() { - return keyspace; - } - - public void setKeyspace(String keyspace) { - this.keyspace = keyspace; - } - - public Integer getPort() { - return port; - } - - public void setPort(Integer port) { - this.port = port; + public void setTableName(String tableName) { + this.tableName = tableName; } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java index 3dbb41d..ff69008 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java @@ -1,4 +1,5 @@ -/***************************************************************************** +/* + ***************************************************************************** * Copyright (c) 2018 Jet Propulsion Laboratory, * California Institute of Technology. All rights reserved *****************************************************************************/ @@ -8,14 +9,14 @@ package gov.nasa.jpl.nexus.ningester.writer.properties; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; -@ConfigurationProperties("solr") +@ConfigurationProperties @Component("solrStoreProperties") public class SolrStore { - private String url; - private String zkHost; - private String collection; + private Integer commitWithin = 1000; + private Integer geoPrecision = 3; + private String collection = "nexustiles"; public String getCollection() { @@ -26,19 +27,19 @@ public class SolrStore { this.collection = collection; } - public String getZkHost() { - return zkHost; + public Integer getCommitWithin() { + return commitWithin; } - public void setZkHost(String zkHost) { - this.zkHost = zkHost; + public void setCommitWithin(Integer commitWithin) { + this.commitWithin = commitWithin; } - public String getUrl() { - return url; + public Integer getGeoPrecision() { + return geoPrecision; } - public void setUrl(String url) { - this.url = url; + public void setGeoPrecision(Integer geoPrecision) { + this.geoPrecision = geoPrecision; } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/main/resources/application.yml ---------------------------------------------------------------------- diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index b536524..db5b4ad 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,5 +1,9 @@ spring: + profiles: default autoconfigure: exclude: - org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration - - org.springframework.boot.autoconfigure.data.cassandra.CassandraDataAutoConfiguration \ No newline at end of file + - org.springframework.boot.autoconfigure.data.cassandra.CassandraDataAutoConfiguration + - org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration + + http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java index 564950f..d677a45 100644 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java @@ -6,8 +6,6 @@ import org.springframework.batch.item.ExecutionContext; import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.Resource; -import java.math.BigDecimal; -import java.math.RoundingMode; import java.util.ArrayList; import java.util.Arrays; import java.util.List; http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/test/java/gov/nasa/jpl/nexus/ningester/writer/SolrStoreTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/writer/SolrStoreTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/writer/SolrStoreTest.java index a7fee32..b8c2797 100644 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/writer/SolrStoreTest.java +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/writer/SolrStoreTest.java @@ -8,7 +8,6 @@ package gov.nasa.jpl.nexus.ningester.writer; import org.apache.solr.common.SolrInputDocument; import org.junit.Test; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; -import org.springframework.mock.env.MockEnvironment; import static org.junit.Assert.assertEquals; @@ -17,7 +16,6 @@ public class SolrStoreTest { @Test public void testGetSolrDocFromTileSummary() { SolrStore solrStore = new SolrStore(null); - solrStore.setEnvironment(new MockEnvironment()); NexusContent.TileSummary tileSummary = NexusContent.TileSummary.newBuilder() .setTileId("1") http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java ---------------------------------------------------------------------- diff --git a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java index 397a62d..b5c4d9f 100644 --- a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java +++ b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java @@ -6,27 +6,35 @@ package gov.nasa.jpl.nexus.ningester.testjobs; -import org.junit.Assert; +import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperties; +import gov.nasa.jpl.nexus.ningester.configuration.properties.DatasourceProperties; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersBuilder; -import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; +import org.springframework.data.cassandra.core.CassandraTemplate; +import org.springframework.data.solr.core.SolrTemplate; +import org.springframework.data.solr.core.query.SimpleQuery; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import static gov.nasa.jpl.nexus.ningester.testjobs.TestUtils.assertEqualsEventually; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + @RunWith(SpringRunner.class) @SpringBootTest @TestPropertySource(properties = {"spring.config.location = classpath:testjobs/AvhrrJobTest.yml"}) -@ActiveProfiles({"test"}) +@ActiveProfiles({"test", "cassandra", "solr"}) public class AvhrrJobTest { @TestConfiguration @@ -37,16 +45,29 @@ public class AvhrrJobTest { return new JobLauncherTestUtils(); } - @Bean - ItemWriter<NexusContent.NexusTile> writer() { - return items -> System.out.println("Wrote " + items.size() + " item(s)."); - } - } @Autowired JobLauncherTestUtils jobLauncherTestUtils; + @Autowired + CassandraTemplate cassandraTemplate; + + @Autowired + SolrTemplate solrTemplate; + + @Autowired + DatasourceProperties datasourceProperties; + + @Autowired + ApplicationProperties applicationProperties; + + @Before + public void emptyDatabase() { + solrTemplate.delete(datasourceProperties.getSolrStore().getCollection(), new SimpleQuery("*:*")); + cassandraTemplate.truncate(datasourceProperties.getCassandraStore().getTableName()); + } + @Test public void testJobCompletes() throws Exception { @@ -57,6 +78,21 @@ public class AvhrrJobTest { JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - Assert.assertEquals("COMPLETED", jobExecution.getExitStatus().getExitCode()); + assertThat(jobExecution.getExitStatus().getExitCode(), is("COMPLETED")); + StepExecution stepExecution = jobExecution.getStepExecutions().iterator().next(); + assertThat(stepExecution.getReadCount(), is(5184)); + assertThat(stepExecution.getWriteCount(), is(3904)); + assertThat(stepExecution.getFilterCount(), is(1280)); + + assertEqualsEventually(3904L, + () -> solrTemplate.count(datasourceProperties.getSolrStore().getCollection(), + new SimpleQuery("dataset_s: " + applicationProperties.getAddDatasetName().getDatasetName())), + 3); + + long cassandraCount = cassandraTemplate.count(datasourceProperties.getCassandraStore().getTableName()); + + assertThat(cassandraCount, is(3904L)); } + + } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java ---------------------------------------------------------------------- diff --git a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java index 45f94aa..fe9ba2d 100644 --- a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java +++ b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java @@ -5,27 +5,35 @@ package gov.nasa.jpl.nexus.ningester.testjobs; -import org.junit.Assert; +import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperties; +import gov.nasa.jpl.nexus.ningester.configuration.properties.DatasourceProperties; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersBuilder; -import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.core.StepExecution; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; +import org.springframework.data.cassandra.core.CassandraTemplate; +import org.springframework.data.solr.core.SolrTemplate; +import org.springframework.data.solr.core.query.SimpleQuery; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import static gov.nasa.jpl.nexus.ningester.testjobs.TestUtils.assertEqualsEventually; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + @RunWith(SpringRunner.class) @SpringBootTest -@TestPropertySource(properties = { "spring.config.location = classpath:testjobs/SmapJobTest.yml" }) -@ActiveProfiles({"test"}) +@TestPropertySource(properties = {"spring.config.location = classpath:testjobs/SmapJobTest.yml"}) +@ActiveProfiles({"test", "cassandra", "solr"}) public class SmapJobTest { @TestConfiguration @@ -36,16 +44,28 @@ public class SmapJobTest { return new JobLauncherTestUtils(); } - @Bean - ItemWriter<NexusContent.NexusTile> writer() { - return items -> System.out.println("Wrote " + items.size() + " item(s)."); - } - } @Autowired JobLauncherTestUtils jobLauncherTestUtils; + @Autowired + CassandraTemplate cassandraTemplate; + + @Autowired + SolrTemplate solrTemplate; + + @Autowired + DatasourceProperties datasourceProperties; + + @Autowired + ApplicationProperties applicationProperties; + + @Before + public void emptyDatabase() { + solrTemplate.delete(datasourceProperties.getSolrStore().getCollection(), new SimpleQuery("*:*")); + cassandraTemplate.truncate(datasourceProperties.getCassandraStore().getTableName()); + } @Test public void testJobCompletes() throws Exception { @@ -56,6 +76,21 @@ public class SmapJobTest { JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - Assert.assertEquals("COMPLETED", jobExecution.getExitStatus().getExitCode()); + assertThat(jobExecution.getExitStatus().getExitCode(), is("COMPLETED")); + StepExecution stepExecution = jobExecution.getStepExecutions().iterator().next(); + assertThat(stepExecution.getReadCount(), is(1624)); + assertThat(stepExecution.getWriteCount(), is(990)); + assertThat(stepExecution.getFilterCount(), is(634)); + + assertEqualsEventually(990L, + () -> solrTemplate.count(datasourceProperties.getSolrStore().getCollection(), + new SimpleQuery("dataset_s: " + applicationProperties.getAddDatasetName().getDatasetName())), + 2); + + long cassandraCount = cassandraTemplate.count(datasourceProperties.getCassandraStore().getTableName()); + + assertThat(cassandraCount, is(990L)); + + } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/TestUtils.java ---------------------------------------------------------------------- diff --git a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/TestUtils.java b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/TestUtils.java new file mode 100644 index 0000000..6ce6850 --- /dev/null +++ b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/TestUtils.java @@ -0,0 +1,28 @@ +/* + **************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.testjobs; + +import java.util.function.Supplier; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +public class TestUtils { + + public static <T> void assertEqualsEventually(T expected, Supplier<T> actualSupplier, int secondsToWait) throws InterruptedException { + int n = 0; + int iterations = 10 * secondsToWait; + int sleepTimeMillis = 100; + + T suppliedValue = actualSupplier.get(); + while (!suppliedValue.equals(expected) && n++ < iterations) { + Thread.sleep(sleepTimeMillis); + suppliedValue = actualSupplier.get(); + } + assertThat("Did not equal after waiting " + (iterations * sleepTimeMillis / 1000) + " seconds.", suppliedValue, is(expected)); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/testJobs/resources/testjobs/AvhrrJobTest.yml ---------------------------------------------------------------------- diff --git a/src/testJobs/resources/testjobs/AvhrrJobTest.yml b/src/testJobs/resources/testjobs/AvhrrJobTest.yml index 05ed43d..a6b7e4e 100644 --- a/src/testJobs/resources/testjobs/AvhrrJobTest.yml +++ b/src/testJobs/resources/testjobs/AvhrrJobTest.yml @@ -1,7 +1,3 @@ -spring: - profiles: test - ---- # Tile Slicer Config ningester: tile_slicer: sliceFileByTilesDesired @@ -21,8 +17,6 @@ ningester: - addDatasetName pythonChainProcessor: enabled: - base_url: http://127.0.0.1:5000/ - uri_path: processorchain processor_list: - name: GridReadingProcessor @@ -48,10 +42,24 @@ ningester: tile_writer: data_store: cassandraStore metadata_store: solrStore - cassandraStore: - cassandraContactPoints: 127.0.0.1 - cassandraKeyspace: nexustiles - cassandraPort: 9042 - solrStore: - solrUrl: http://127.0.0.1:8983 - solrCollection: nexustiles \ No newline at end of file +--- +# Connection settings for the test profile +spring: + profiles: test + data: + cassandra: + keyspaceName: nexustiles + contactPoints: 127.0.0.1 + solr: + host: http://127.0.0.1:8983/solr/ + +ningester: + pythonChainProcessor: + base_url: http://127.0.0.1:5000/ + uri_path: processorchain + +datasource: + solrStore: + commitWithin: 1500 + geoPrecision: 3 + collection: nexustiles \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/787b59fa/src/testJobs/resources/testjobs/SmapJobTest.yml ---------------------------------------------------------------------- diff --git a/src/testJobs/resources/testjobs/SmapJobTest.yml b/src/testJobs/resources/testjobs/SmapJobTest.yml index e8fda0d..6a72c6c 100644 --- a/src/testJobs/resources/testjobs/SmapJobTest.yml +++ b/src/testJobs/resources/testjobs/SmapJobTest.yml @@ -1,57 +1,67 @@ -spring: - profiles: test - ---- # Tile Slicer Config ningester: - tile_slicer: sliceFileByDimension - sliceFileByDimension: - sliceByDimension: 1 - dimensionNamePrefix: phony_dim_ - dimensions: - - 0 - - 1 + tile_slicer: sliceFileByDimension + sliceFileByDimension: + sliceByDimension: 1 + dimensionNamePrefix: phony_dim_ + dimensions: + - 0 + - 1 --- # Tile processors configuration ningester: - tile_processors: - - pythonChainProcessor - - generateTileId - - addDatasetName - pythonChainProcessor: - enabled: - base_url: http://127.0.0.1:5000/ - uri_path: processorchain - processor_list: - - - name: SwathReadingProcessor - config: - latitude: lat - longitude: lon - time: row_time - variable_to_read: smap_sss - glblattr_day: REV_START_TIME - glblattr_day_format: '%Y-%jT%H:%M:%S.%f' - - - name: EmptyTileFilter - - - name: TileSummarizingProcessor - generateTileId: - enabled: - addDatasetName: - enabled: - datasetName: SMAP_L2B_SSS + tile_processors: + - pythonChainProcessor + - generateTileId + - addDatasetName + pythonChainProcessor: + enabled: + processor_list: + - + name: SwathReadingProcessor + config: + latitude: lat + longitude: lon + time: row_time + variable_to_read: smap_sss + glblattr_day: REV_START_TIME + glblattr_day_format: '%Y-%jT%H:%M:%S.%f' + - + name: EmptyTileFilter + - + name: TileSummarizingProcessor + generateTileId: + enabled: + addDatasetName: + enabled: + datasetName: SMAP_L2B_SSS --- # Tile writer configuration ningester: tile_writer: data_store: cassandraStore metadata_store: solrStore - cassandraStore: - cassandraContactPoints: 127.0.0.1 - cassandraKeyspace: nexustiles - cassandraPort: 9042 - solrStore: - solrUrl: http://127.0.0.1:8983 - solrCollection: nexustiles \ No newline at end of file +--- +# Connection settings for the test profile +spring: + profiles: test + data: + cassandra: + keyspaceName: nexustiles + contactPoints: 127.0.0.1 + solr: + host: http://127.0.0.1:8983/solr/ + +ningester: + pythonChainProcessor: + base_url: http://127.0.0.1:5000/ + uri_path: processorchain + +datasource: + solrStore: + commitWithin: 1500 + geoPrecision: 3 + collection: nexustiles + cassandraStore: + tableName: sea_surface_temp \ No newline at end of file
