Removed batch configuration from AppConfig. Organized imports
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/0ba0dd44 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/0ba0dd44 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/0ba0dd44 Branch: refs/heads/master Commit: 0ba0dd44f7f3ecae354e146342ea0a3ce3d8bff2 Parents: 8437400 Author: Frank Greguska <[email protected]> Authored: Wed Jan 17 17:46:23 2018 -0800 Committer: Frank Greguska <[email protected]> Committed: Wed Jan 17 17:46:23 2018 -0800 ---------------------------------------------------------------------- .../nexus/ningester/NingesterApplication.java | 6 +-- .../ningester/configuration/AppConfig.java | 26 ++++------- .../ningester/configuration/BatchConfig.java | 49 ++++++++++++++++---- .../configuration/DatasourceConfig.java | 18 +------ .../properties/ApplicationProperties.java | 12 +---- .../ningester/datatiler/NetCDFItemReader.java | 8 +--- .../datatiler/SliceFileByDimension.java | 2 +- .../datatiler/SliceFileByTilesDesired.java | 4 +- .../processors/AddTimeFromGranuleName.java | 12 ++--- .../ningester/processors/GenerateTileId.java | 2 +- .../processors/PythonChainProcessor.java | 13 +----- .../nexus/ningester/writer/CassandraStore.java | 3 +- .../jpl/nexus/ningester/writer/DynamoStore.java | 1 - .../jpl/nexus/ningester/writer/NexusWriter.java | 3 +- .../jpl/nexus/ningester/writer/S3Store.java | 2 - .../jpl/nexus/ningester/writer/SolrStore.java | 17 +++---- .../nexus/ningester/testjobs/AvhrrJobTest.java | 25 ++++------ .../nexus/ningester/testjobs/SmapJobTest.java | 24 ++++------ 18 files changed, 96 insertions(+), 131 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/NingesterApplication.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/NingesterApplication.java b/src/main/java/gov/nasa/jpl/nexus/ningester/NingesterApplication.java index 5945596..e4133dd 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/NingesterApplication.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/NingesterApplication.java @@ -6,7 +6,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; @SpringBootApplication public class NingesterApplication { - public static void main(String[] args) { - SpringApplication.run(NingesterApplication.class, args); - } + public static void main(String[] args) { + SpringApplication.run(NingesterApplication.class, args); + } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java index f30b627..05b6e32 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java @@ -15,21 +15,17 @@ import gov.nasa.jpl.nexus.ningester.writer.DataStore; import gov.nasa.jpl.nexus.ningester.writer.MetadataStore; import gov.nasa.jpl.nexus.ningester.writer.NexusWriter; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; -import org.springframework.batch.core.configuration.annotation.JobScope; -import org.springframework.batch.item.ItemProcessor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.core.io.Resource; import org.springframework.http.MediaType; import org.springframework.http.converter.HttpMessageConverter; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.DefaultUriTemplateHandler; -import java.io.IOException; import java.util.Collections; import java.util.List; @@ -118,47 +114,45 @@ public class AppConfig { */ @Bean @ConditionalOnProperty(prefix = "ningester.addDatasetName", name = "enabled") - protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> addDatasetName() { + protected AddDatasetName addDatasetNameBean() { AddDatasetName processor = new AddDatasetName(applicationProperties.getAddDatasetName().getDatasetName()); - return processor::addDatasetName; + return processor; } @Bean @ConditionalOnProperty(prefix = "ningester.addDayOfYearAttribute", name = "enabled") - protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> addDayOfYearAttribute() { + protected AddDayOfYearAttribute addDayOfYearAttributeBean() { AddDayOfYearAttribute processor = new AddDayOfYearAttribute(applicationProperties.getAddDayOfYearAttribute().getRegex()); - return processor::setDayOfYearFromGranuleName; + return processor; } @Bean @ConditionalOnProperty(prefix = "ningester.addTimeFromGranuleName", name = "enabled") - protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> addTimeFromGranuleName() { + protected AddTimeFromGranuleName addTimeFromGranuleNameBean() { AddTimeFromGranuleName processor = new AddTimeFromGranuleName(applicationProperties.getAddTimeFromGranuleName().getRegex(), applicationProperties.getAddTimeFromGranuleName().getDateFormat()); - return processor::setTimeFromGranuleName; + return processor; } @Bean @ConditionalOnProperty(prefix = "ningester.generateTileId", name = "enabled") - protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> generateTileId() { + protected GenerateTileId generateTileIdBean() { GenerateTileId processor = new GenerateTileId(); processor.setSalt(applicationProperties.getGenerateTileId().getSalt()); - return processor::addTileId; + return processor; } @Bean - @JobScope @ConditionalOnProperty(prefix = "ningester.pythonChainProcessor", name = "enabled") - protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> pythonChainProcessor(RestTemplate restTemplate, Resource granule) throws IOException { + protected PythonChainProcessor pythonChainProcessorBean(RestTemplate restTemplate) { PythonChainProcessor processor = new PythonChainProcessor(restTemplate); - processor.setGranule(granule); processor.setProcessorList(applicationProperties.getPythonChainProcessor().getProcessorList()); processor.setUriPath(applicationProperties.getPythonChainProcessor().getUriPath()); - return processor::nexusTileProcessor; + return processor; } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java index 5bd108f..8a5504e 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java @@ -3,9 +3,7 @@ package gov.nasa.jpl.nexus.ningester.configuration; import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperties; import gov.nasa.jpl.nexus.ningester.datatiler.FileSlicer; import gov.nasa.jpl.nexus.ningester.datatiler.NetCDFItemReader; -import gov.nasa.jpl.nexus.ningester.processors.CompositeItemProcessor; -import gov.nasa.jpl.nexus.ningester.writer.DataStore; -import gov.nasa.jpl.nexus.ningester.writer.MetadataStore; +import gov.nasa.jpl.nexus.ningester.processors.*; import gov.nasa.jpl.nexus.ningester.writer.NexusWriter; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.batch.core.Job; @@ -17,10 +15,10 @@ import org.springframework.batch.core.configuration.annotation.StepBuilderFactor import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemStreamReader; import org.springframework.batch.item.ItemWriter; -import org.springframework.batch.item.support.CompositeItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -28,8 +26,6 @@ import org.springframework.context.annotation.Import; import org.springframework.core.io.Resource; import org.springframework.core.io.ResourceLoader; -import java.util.Arrays; - @Configuration @EnableBatchProcessing @Import(AppConfig.class) @@ -48,8 +44,8 @@ public class BatchConfig { protected ApplicationContext context; @Bean - public Job job(@Qualifier("step1") Step step1) { - return jobs.get("testNc4").start(step1).build(); + public Job job(@Qualifier("ingestGranule") Step step1) { + return jobs.get("Ningester").start(step1).build(); } @Bean @@ -80,12 +76,45 @@ public class BatchConfig { @Bean @JobScope - protected Step step1(ItemStreamReader<NexusContent.NexusTile> reader, ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> processor, ItemWriter<NexusContent.NexusTile> writer) { - return steps.get("step1") + protected Step ingestGranule(ItemStreamReader<NexusContent.NexusTile> reader, ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> processor, ItemWriter<NexusContent.NexusTile> writer) { + return steps.get("ingestGranule") .<NexusContent.NexusTile, NexusContent.NexusTile>chunk(10) .reader(reader) .processor(processor) .writer(writer).build(); } + /* + * Item Processor beans defined below + */ + @Bean + @ConditionalOnBean(AddDatasetName.class) + protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> addDatasetName(AddDatasetName addDatasetNameBean) { + return addDatasetNameBean::addDatasetName; + } + + @Bean + @ConditionalOnBean(AddDayOfYearAttribute.class) + protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> addDayOfYearAttribute(AddDayOfYearAttribute addDayOfYearAttributeBean) { + return addDayOfYearAttributeBean::setDayOfYearFromGranuleName; + } + + @Bean + @ConditionalOnBean(AddTimeFromGranuleName.class) + protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> addTimeFromGranuleName(AddTimeFromGranuleName addTimeFromGranuleNameBean) { + return addTimeFromGranuleNameBean::setTimeFromGranuleName; + } + + @Bean + @ConditionalOnBean(GenerateTileId.class) + protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> generateTileId(GenerateTileId generateTileIdBean) { + return generateTileIdBean::addTileId; + } + + @Bean + @ConditionalOnBean(PythonChainProcessor.class) + protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> pythonChainProcessor(PythonChainProcessor pythonChainProcessorBean) { + return pythonChainProcessorBean::nexusTileProcessor; + } + } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java index 8d52f59..7eee744 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java @@ -5,40 +5,24 @@ import com.amazonaws.regions.Regions; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; import com.amazonaws.services.s3.AmazonS3Client; -import com.datastax.driver.core.ProtocolVersion; import gov.nasa.jpl.nexus.ningester.configuration.properties.DatasourceProperties; import gov.nasa.jpl.nexus.ningester.writer.*; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration; -import org.springframework.boot.autoconfigure.cassandra.ClusterBuilderCustomizer; import org.springframework.boot.autoconfigure.data.cassandra.CassandraDataAutoConfiguration; import org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration; -import org.springframework.boot.autoconfigure.solr.SolrProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Profile; -import org.springframework.core.env.Environment; -import org.springframework.data.cassandra.config.CassandraClusterFactoryBean; -import org.springframework.data.cassandra.config.CassandraSessionFactoryBean; -import org.springframework.data.cassandra.config.SchemaAction; -import org.springframework.data.cassandra.convert.CassandraConverter; -import org.springframework.data.cassandra.convert.MappingCassandraConverter; -import org.springframework.data.cassandra.core.CassandraOperations; import org.springframework.data.cassandra.core.CassandraTemplate; -import org.springframework.data.cassandra.mapping.BasicCassandraMappingContext; -import org.springframework.data.cassandra.mapping.CassandraMappingContext; import org.springframework.data.solr.core.SolrOperations; import org.springframework.data.solr.core.SolrTemplate; import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; import javax.sql.DataSource; -import java.util.Collections; -import java.util.List; @Configuration public class DatasourceConfig { @@ -121,7 +105,7 @@ public class DatasourceConfig { @Bean public MetadataStore metadataStore(SolrOperations solrTemplate) { - SolrStore solrStore = new SolrStore(solrTemplate); + SolrStore solrStore = new SolrStore(solrTemplate); solrStore.setCollection(datasourceProperties.getSolrStore().getCollection()); solrStore.setCommitWithin(datasourceProperties.getSolrStore().getCommitWithin()); solrStore.setGeoPrecision(datasourceProperties.getSolrStore().getGeoPrecision()); http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java index 9dace2e..ced72ee 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java @@ -19,30 +19,22 @@ import java.util.List; @Component public class ApplicationProperties { - private String tileSlicer; - @NestedConfigurationProperty private final SliceFileByDimension sliceFileByDimension = new SliceFileByDimension(); - @NestedConfigurationProperty private final SliceFileByTilesDesired sliceFileByTilesDesired = new SliceFileByTilesDesired(); - - private List<String> tileProcessors = new ArrayList<>(); - @NestedConfigurationProperty private final AddDayOfYearAttribute addDayOfYearAttribute = new AddDayOfYearAttribute(); - @NestedConfigurationProperty private final AddTimeFromGranuleName addTimeFromGranuleName = new AddTimeFromGranuleName(); - @NestedConfigurationProperty private final GenerateTileId generateTileId = new GenerateTileId(); - @NestedConfigurationProperty private final PythonChainProcessor pythonChainProcessor = new PythonChainProcessor(); - @NestedConfigurationProperty private final AddDatasetName addDatasetName = new AddDatasetName(); + private String tileSlicer; + private List<String> tileProcessors = new ArrayList<>(); public PythonChainProcessor getPythonChainProcessor() { return pythonChainProcessor; http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java index 4a018fb..9ca87a5 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java @@ -5,14 +5,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.item.ExecutionContext; import org.springframework.batch.item.ItemStreamException; -import org.springframework.batch.item.NonTransientResourceException; import org.springframework.batch.item.UnexpectedInputException; import org.springframework.batch.item.file.ResourceAwareItemReaderItemStream; import org.springframework.core.io.Resource; import ucar.nc2.dataset.NetcdfDataset; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.net.URL; import java.util.List; @@ -21,10 +19,8 @@ import java.util.stream.IntStream; public class NetCDFItemReader implements ResourceAwareItemReaderItemStream<NexusContent.NexusTile> { - private static final Logger log = LoggerFactory.getLogger(NetCDFItemReader.class); - static final String CURRENT_TILE_SPEC_INDEX_KEY = "current.tile.spec.index"; - + private static final Logger log = LoggerFactory.getLogger(NetCDFItemReader.class); private List<String> tileSpecList; private Integer currentTileSpecIndex; @@ -43,7 +39,7 @@ public class NetCDFItemReader implements ResourceAwareItemReaderItemStream<Nexus @Override public NexusContent.NexusTile read() { - if(this.currentTileSpecIndex == this.tileSpecList.size()){ + if (this.currentTileSpecIndex == this.tileSpecList.size()) { //End of stream return null; } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java index 0a5b594..d431397 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java @@ -51,7 +51,7 @@ public class SliceFileByDimension implements FileSlicer { // This is sort of a hack to help the python netcdf library. When you try to get dimensions by name and they are unnamed, the // python library uses 'phony_dim_' then the index of the dimension as the dimension name. Weird, I know. - if(Strings.isNullOrEmpty(this.dimensionNamePrefix)){ + if (Strings.isNullOrEmpty(this.dimensionNamePrefix)) { this.dimensionNamePrefix = "phony_dim_"; } Map<String, Integer> dimensionNameToLength; http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java index b1ca335..05a2b45 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java @@ -82,14 +82,14 @@ public class SliceFileByTilesDesired implements FileSlicer { List<String> addTimeDimension(List<String> specs, Integer timeLen) { - if(timeLen > 0) { + if (timeLen > 0) { return specs.stream().map(sectionSpec -> IntStream.range(0, timeLen) .mapToObj(timeIndex -> this.timeDimension + ":" + timeIndex + ":" + (timeIndex + 1) + "," + sectionSpec) .collect(Collectors.toList())) .flatMap(List::stream) .collect(Collectors.toList()); - }else { + } else { return specs; } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeFromGranuleName.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeFromGranuleName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeFromGranuleName.java index a83e6e3..0f53a1c 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeFromGranuleName.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeFromGranuleName.java @@ -1,8 +1,8 @@ /* -***************************************************************************** -* Copyright (c) 2018 Jet Propulsion Laboratory, -* California Institute of Technology. All rights reserved -*****************************************************************************/ + ***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ package gov.nasa.jpl.nexus.ningester.processors; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; @@ -16,7 +16,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -public class AddTimeFromGranuleName{ +public class AddTimeFromGranuleName { private Pattern regex; @@ -32,7 +32,7 @@ public class AddTimeFromGranuleName{ NexusContent.NexusTile.Builder outTileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(inputTile); - switch(inputTile.getTile().getTileTypeCase()){ + switch (inputTile.getTile().getTileTypeCase()) { case GRID_TILE: String granuleName = inputTile.getSummary().getGranule(); http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java index 6d00a99..dfa69ac 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java @@ -23,7 +23,7 @@ public class GenerateTileId { this.salt = salt; } - public NexusContent.NexusTile addTileId(NexusContent.NexusTile inputTile){ + public NexusContent.NexusTile addTileId(NexusContent.NexusTile inputTile) { NexusContent.NexusTile.Builder outTileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(inputTile); String granuleFileName = inputTile.getSummary().getGranule(); http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java index be6ec78..2c09b87 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java @@ -11,15 +11,12 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import gov.nasa.jpl.nexus.ningester.processors.properties.PythonProcessorModule; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; -import org.springframework.core.io.Resource; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.MediaType; import org.springframework.web.client.RestTemplate; -import java.io.File; -import java.io.IOException; import java.util.Collections; import java.util.List; @@ -31,16 +28,10 @@ public class PythonChainProcessor { private String uriPath; - private File granule; - public PythonChainProcessor(RestTemplate restTemplate) { this.restTemplate = restTemplate; } - public void setGranule(Resource granule) throws IOException { - this.granule = granule.getFile(); - } - public NexusContent.NexusTile nexusTileProcessor(NexusContent.NexusTile nexusTile) { HttpHeaders headers = new HttpHeaders(); @@ -57,13 +48,11 @@ public class PythonChainProcessor { HttpEntity<PythonChainProcessorRequest> requestEntity = new HttpEntity<>(chainProcessorRequest, headers); - NexusContent.NexusTile outNexusTile = restTemplate.exchange( + return restTemplate.exchange( uriPath, HttpMethod.POST, requestEntity, NexusContent.NexusTile.class).getBody(); - - return outNexusTile; } public void setProcessorList(List<PythonProcessorModule> processorList) { http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java index d3522a1..6e9bcd2 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java @@ -11,7 +11,6 @@ import org.springframework.data.cassandra.core.CassandraOperations; import java.nio.ByteBuffer; import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -30,7 +29,7 @@ public class CassandraStore implements DataStore { @Override public void saveData(List<? extends NexusContent.NexusTile> nexusTiles) { - String query = "insert into "+ this.tableName + " (tile_id, tile_blob) VALUES (?, ?)"; + String query = "insert into " + this.tableName + " (tile_id, tile_blob) VALUES (?, ?)"; cassandraTemplate.ingest(query, nexusTiles.stream() .map(nexusTile -> getCassandraRowFromTileData(nexusTile.getTile())) .collect(Collectors.toList())); http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java index 43b041f..ebcc5a6 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java @@ -11,7 +11,6 @@ import com.amazonaws.services.dynamodbv2.document.Item; import com.amazonaws.services.dynamodbv2.document.Table; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; -import java.util.Collection; import java.util.List; /** http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java index 63a98e1..0156b5a 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java @@ -14,11 +14,10 @@ import java.util.List; public class NexusWriter { + private static final Logger log = LoggerFactory.getLogger(NexusWriter.class); private MetadataStore metadataStore; private DataStore dataStore; - private static final Logger log = LoggerFactory.getLogger(NexusWriter.class); - public NexusWriter(MetadataStore metadataStore, DataStore dataStore) { this.metadataStore = metadataStore; this.dataStore = dataStore; http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java index 5998056..2c71521 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java @@ -10,12 +10,10 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; -import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent.NexusTile; import java.io.ByteArrayInputStream; import java.io.InputStream; -import java.util.Collection; import java.util.List; /** http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java index ed91ff5..b3198ea 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java @@ -10,10 +10,8 @@ import org.apache.solr.common.SolrInputField; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.core.env.Environment; import org.springframework.data.solr.core.SolrOperations; -import javax.annotation.Resource; import java.math.BigDecimal; import java.nio.file.Paths; import java.text.SimpleDateFormat; @@ -22,23 +20,20 @@ import java.util.stream.Collectors; public class SolrStore implements MetadataStore { - private Integer commitWithin = 1000; - private Integer geoPrecision = 3; - private String collection = "nexustiles"; - - private SolrOperations solr; - - private Logger log = LoggerFactory.getLogger(SolrStore.class); - //TODO This will be refactored at some point to be dynamic per-message. Or maybe per-group. private static final String TABLE_NAME = "sea_surface_temp"; - private static final SimpleDateFormat iso = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); static { iso.setTimeZone(TimeZone.getTimeZone("UTC")); } + private Integer commitWithin = 1000; + private Integer geoPrecision = 3; + private String collection = "nexustiles"; + private SolrOperations solr; + private Logger log = LoggerFactory.getLogger(SolrStore.class); + public SolrStore(SolrOperations solr) { this.solr = solr; } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java ---------------------------------------------------------------------- diff --git a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java index b5c4d9f..2822629 100644 --- a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java +++ b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java @@ -37,28 +37,14 @@ import static org.hamcrest.MatcherAssert.assertThat; @ActiveProfiles({"test", "cassandra", "solr"}) public class AvhrrJobTest { - @TestConfiguration - static class NingesterApplicationTestsConfig { - - @Bean - JobLauncherTestUtils jobLauncherTestUtils() { - return new JobLauncherTestUtils(); - } - - } - @Autowired JobLauncherTestUtils jobLauncherTestUtils; - @Autowired CassandraTemplate cassandraTemplate; - @Autowired SolrTemplate solrTemplate; - @Autowired DatasourceProperties datasourceProperties; - @Autowired ApplicationProperties applicationProperties; @@ -68,7 +54,6 @@ public class AvhrrJobTest { cassandraTemplate.truncate(datasourceProperties.getCassandraStore().getTableName()); } - @Test public void testJobCompletes() throws Exception { @@ -94,5 +79,15 @@ public class AvhrrJobTest { assertThat(cassandraCount, is(3904L)); } + @TestConfiguration + static class NingesterApplicationTestsConfig { + + @Bean + JobLauncherTestUtils jobLauncherTestUtils() { + return new JobLauncherTestUtils(); + } + + } + } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/0ba0dd44/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java ---------------------------------------------------------------------- diff --git a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java index fe9ba2d..89d01eb 100644 --- a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java +++ b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java @@ -36,28 +36,14 @@ import static org.hamcrest.MatcherAssert.assertThat; @ActiveProfiles({"test", "cassandra", "solr"}) public class SmapJobTest { - @TestConfiguration - static class NingesterApplicationTestsConfig { - - @Bean - JobLauncherTestUtils jobLauncherTestUtils() { - return new JobLauncherTestUtils(); - } - - } - @Autowired JobLauncherTestUtils jobLauncherTestUtils; - @Autowired CassandraTemplate cassandraTemplate; - @Autowired SolrTemplate solrTemplate; - @Autowired DatasourceProperties datasourceProperties; - @Autowired ApplicationProperties applicationProperties; @@ -93,4 +79,14 @@ public class SmapJobTest { } + + @TestConfiguration + static class NingesterApplicationTestsConfig { + + @Bean + JobLauncherTestUtils jobLauncherTestUtils() { + return new JobLauncherTestUtils(); + } + + } }
