getting ready for Apache commit
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/7a5efa30 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/7a5efa30 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/7a5efa30 Branch: refs/heads/master Commit: 7a5efa309543b8d1cb53d748aaf5de0ae93c5568 Parents: 38b7076 Author: Frank Greguska <[email protected]> Authored: Mon Jan 22 16:16:23 2018 -0800 Committer: Frank Greguska <[email protected]> Committed: Mon Jan 22 16:16:23 2018 -0800 ---------------------------------------------------------------------- build.gradle | 24 ++- .../nexus/ningester/NingesterApplication.java | 15 -- .../ningester/configuration/AppConfig.java | 158 --------------- .../ningester/configuration/BatchConfig.java | 122 ------------ .../configuration/DatasourceConfig.java | 116 ----------- .../properties/ApplicationProperties.java | 78 -------- .../properties/DatasourceProperties.java | 48 ----- .../nexus/ningester/datatiler/FileSlicer.java | 16 -- .../ningester/datatiler/NetCDFItemReader.java | 124 ------------ .../datatiler/SliceFileByDimension.java | 117 ----------- .../datatiler/SliceFileByTilesDesired.java | 104 ---------- .../properties/SliceFileByDimension.java | 44 ----- .../properties/SliceFileByTilesDesired.java | 44 ----- .../ningester/http/NexusTileConverter.java | 40 ---- .../ningester/processors/AddDatasetName.java | 28 --- .../processors/AddDayOfYearAttribute.java | 71 ------- .../processors/AddTimeFromGranuleName.java | 74 ------- .../processors/CompositeItemProcessor.java | 42 ---- .../ningester/processors/GenerateTileId.java | 48 ----- .../processors/PythonChainProcessor.java | 91 --------- .../processors/properties/AddDatasetName.java | 24 --- .../properties/AddDayOfYearAttribute.java | 25 --- .../properties/AddTimeFromGranuleName.java | 33 ---- .../processors/properties/GenerateTileId.java | 24 --- .../properties/PythonChainProcessor.java | 42 ---- .../properties/PythonProcessorModule.java | 27 --- .../nexus/ningester/writer/CassandraStore.java | 48 ----- .../jpl/nexus/ningester/writer/DataStore.java | 16 -- .../ningester/writer/DataStoreException.java | 28 --- .../jpl/nexus/ningester/writer/DynamoStore.java | 54 ------ .../nexus/ningester/writer/MetadataStore.java | 17 -- .../jpl/nexus/ningester/writer/NexusWriter.java | 44 ----- .../jpl/nexus/ningester/writer/S3Store.java | 62 ------ .../jpl/nexus/ningester/writer/SolrStore.java | 166 ---------------- .../writer/properties/CassandraStore.java | 25 --- .../writer/properties/DynamoStore.java | 44 ----- .../ningester/writer/properties/S3Store.java | 34 ---- .../ningester/writer/properties/SolrStore.java | 45 ----- .../sdap/ningester/NingesterApplication.java | 31 +++ .../sdap/ningester/configuration/AppConfig.java | 171 +++++++++++++++++ .../ningester/configuration/BatchConfig.java | 138 +++++++++++++ .../configuration/DatasourceConfig.java | 132 +++++++++++++ .../properties/ApplicationProperties.java | 91 +++++++++ .../properties/DatasourceProperties.java | 60 ++++++ .../sdap/ningester/datatiler/FileSlicer.java | 29 +++ .../ningester/datatiler/NetCDFItemReader.java | 140 ++++++++++++++ .../datatiler/SliceFileByDimension.java | 129 +++++++++++++ .../datatiler/SliceFileByTilesDesired.java | 117 +++++++++++ .../properties/SliceFileByDimension.java | 57 ++++++ .../properties/SliceFileByTilesDesired.java | 57 ++++++ .../sdap/ningester/http/NexusTileConverter.java | 53 +++++ .../ningester/processors/AddDatasetName.java | 40 ++++ .../processors/AddDayOfYearAttribute.java | 83 ++++++++ .../processors/AddTimeFromGranuleName.java | 86 +++++++++ .../processors/CompositeItemProcessor.java | 54 ++++++ .../ningester/processors/GenerateTileId.java | 60 ++++++ .../processors/PythonChainProcessor.java | 103 ++++++++++ .../processors/properties/AddDatasetName.java | 37 ++++ .../properties/AddDayOfYearAttribute.java | 38 ++++ .../properties/AddTimeFromGranuleName.java | 46 +++++ .../processors/properties/GenerateTileId.java | 37 ++++ .../properties/PythonChainProcessor.java | 55 ++++++ .../properties/PythonProcessorModule.java | 40 ++++ .../sdap/ningester/writer/CassandraStore.java | 60 ++++++ .../apache/sdap/ningester/writer/DataStore.java | 28 +++ .../ningester/writer/DataStoreException.java | 40 ++++ .../sdap/ningester/writer/DynamoStore.java | 66 +++++++ .../sdap/ningester/writer/MetadataStore.java | 30 +++ .../sdap/ningester/writer/NexusWriter.java | 55 ++++++ .../apache/sdap/ningester/writer/S3Store.java | 75 ++++++++ .../apache/sdap/ningester/writer/SolrStore.java | 179 +++++++++++++++++ .../writer/properties/CassandraStore.java | 36 ++++ .../writer/properties/DynamoStore.java | 57 ++++++ .../ningester/writer/properties/S3Store.java | 47 +++++ .../ningester/writer/properties/SolrStore.java | 57 ++++++ .../ningester/NingesterApplicationTests.java | 16 -- .../datatiler/NetCDFItemReaderTest.java | 80 -------- .../datatiler/SliceFileByDimensionTest.java | 96 ---------- .../datatiler/SliceFileByTilesDesiredTest.java | 179 ----------------- .../processors/AddDatasetNameTest.java | 31 --- .../processors/AddDayOfYearAttributeTest.java | 84 -------- .../processors/AddTimeFromGranuleNameTest.java | 85 -------- .../processors/GenerateTileIdTest.java | 65 ------- .../nexus/ningester/writer/SolrStoreTest.java | 137 ------------- .../ningester/NingesterApplicationTests.java | 32 ++++ .../datatiler/NetCDFItemReaderTest.java | 96 ++++++++++ .../datatiler/SliceFileByDimensionTest.java | 105 ++++++++++ .../datatiler/SliceFileByTilesDesiredTest.java | 192 +++++++++++++++++++ .../processors/AddDatasetNameTest.java | 43 +++++ .../processors/AddDayOfYearAttributeTest.java | 97 ++++++++++ .../processors/AddTimeFromGranuleNameTest.java | 98 ++++++++++ .../processors/GenerateTileIdTest.java | 78 ++++++++ .../sdap/ningester/writer/SolrStoreTest.java | 150 +++++++++++++++ .../nexus/ningester/testjobs/AvhrrJobTest.java | 93 --------- .../nexus/ningester/testjobs/SmapJobTest.java | 92 --------- .../jpl/nexus/ningester/testjobs/TestUtils.java | 28 --- .../sdap/ningester/testjobs/AvhrrJobTest.java | 105 ++++++++++ .../sdap/ningester/testjobs/SmapJobTest.java | 104 ++++++++++ .../sdap/ningester/testjobs/TestUtils.java | 38 ++++ 99 files changed, 3767 insertions(+), 3133 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index 178f6dc..faa850e 100644 --- a/build.gradle +++ b/build.gradle @@ -1,8 +1,18 @@ -/* - ***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ plugins { id 'java' @@ -16,10 +26,6 @@ plugins { id 'net.researchgate.release' version '2.6.0' } -//group = 'gov.nasa.jpl.nexus.ningester' -//version = '0.0.1-SNAPSHOT' -//sourceCompatibility = 1.8 - repositories { mavenCentral() maven { http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/NingesterApplication.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/NingesterApplication.java b/src/main/java/gov/nasa/jpl/nexus/ningester/NingesterApplication.java deleted file mode 100644 index 09ee307..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/NingesterApplication.java +++ /dev/null @@ -1,15 +0,0 @@ -package gov.nasa.jpl.nexus.ningester; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.ApplicationContext; - -@SpringBootApplication -public class NingesterApplication { - - public static void main(String[] args) { - - ApplicationContext context = SpringApplication.run(NingesterApplication.class, args); - SpringApplication.exit(context); - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java deleted file mode 100644 index b653ab3..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java +++ /dev/null @@ -1,158 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.configuration; - -import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperties; -import gov.nasa.jpl.nexus.ningester.datatiler.FileSlicer; -import gov.nasa.jpl.nexus.ningester.datatiler.SliceFileByDimension; -import gov.nasa.jpl.nexus.ningester.datatiler.SliceFileByTilesDesired; -import gov.nasa.jpl.nexus.ningester.http.NexusTileConverter; -import gov.nasa.jpl.nexus.ningester.processors.*; -import gov.nasa.jpl.nexus.ningester.writer.DataStore; -import gov.nasa.jpl.nexus.ningester.writer.MetadataStore; -import gov.nasa.jpl.nexus.ningester.writer.NexusWriter; -import org.apache.sdap.nexusproto.NexusTile; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.http.MediaType; -import org.springframework.http.converter.HttpMessageConverter; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.util.DefaultUriTemplateHandler; - -import java.util.Collections; -import java.util.List; - -@Configuration -@EnableConfigurationProperties({ApplicationProperties.class}) -public class AppConfig { - - private final ApplicationProperties applicationProperties; - - @Autowired - public AppConfig(ApplicationProperties applicationProperties) { - this.applicationProperties = applicationProperties; - } - - @Bean - @ConditionalOnProperty(prefix = "ningester", name = "tile_slicer", havingValue = "sliceFileByTilesDesired") - @Qualifier("fileSlicer") - protected FileSlicer sliceFileByTilesDesired() { - SliceFileByTilesDesired fileSlicer = new SliceFileByTilesDesired(); - fileSlicer.setDimensions(applicationProperties.getSliceFileByTilesDesired().getDimensions()); - fileSlicer.setTilesDesired(applicationProperties.getSliceFileByTilesDesired().getTilesDesired()); - fileSlicer.setTimeDimension(applicationProperties.getSliceFileByTilesDesired().getTimeDimension()); - return fileSlicer; - } - - @Bean - @ConditionalOnProperty(prefix = "ningester", name = "tile_slicer", havingValue = "sliceFileByDimension") - @Qualifier("fileSlicer") - protected FileSlicer sliceFileByDimension() { - SliceFileByDimension fileSlicer = new SliceFileByDimension(); - fileSlicer.setDimensions(applicationProperties.getSliceFileByDimension().getDimensions()); - fileSlicer.setSliceByDimension(applicationProperties.getSliceFileByDimension().getSliceByDimension()); - fileSlicer.setDimensionNamePrefix(applicationProperties.getSliceFileByDimension().getDimensionNamePrefix()); - return fileSlicer; - } - - @Bean - protected HttpMessageConverter nexusTileConverter() { - NexusTileConverter converter = new NexusTileConverter(); - converter.setSupportedMediaTypes(Collections.singletonList(MediaType.APPLICATION_OCTET_STREAM)); - return converter; - } - - @Bean - @ConditionalOnProperty(prefix = "ningester.pythonChainProcessor", name = "enabled") - protected RestTemplate restTemplate(HttpMessageConverter nexusTileConverter) { - RestTemplate template = new RestTemplate(); - - DefaultUriTemplateHandler uriTemplateHandler = new DefaultUriTemplateHandler(); - uriTemplateHandler.setBaseUrl(applicationProperties.getPythonChainProcessor().getBaseUrl().toString()); - template.setUriTemplateHandler(uriTemplateHandler); - - List<HttpMessageConverter<?>> converters = template.getMessageConverters(); - converters.add(nexusTileConverter); - template.setMessageConverters(converters); - - return template; - } - - @Bean - public MetadataStore metadataStore() { - return new MetadataStore() { - @Override - public void saveMetadata(List<? extends NexusTile> nexusTiles) { - } - - @Override - public void deleteMetadata(List<? extends NexusTile> nexusTiles) { - } - }; - } - - @Bean - public DataStore dataStore() { - return nexusTiles -> { - }; - } - - @Bean - public NexusWriter nexusWriter(MetadataStore metadataStore, DataStore dataStore) { - return new NexusWriter(metadataStore, dataStore); - } - - /* - * Item Processor beans defined below - */ - @Bean - @ConditionalOnProperty(prefix = "ningester.addDatasetName", name = "enabled") - protected AddDatasetName addDatasetNameBean() { - - AddDatasetName processor = new AddDatasetName(applicationProperties.getAddDatasetName().getDatasetName()); - return processor; - } - - @Bean - @ConditionalOnProperty(prefix = "ningester.addDayOfYearAttribute", name = "enabled") - protected AddDayOfYearAttribute addDayOfYearAttributeBean() { - - AddDayOfYearAttribute processor = new AddDayOfYearAttribute(applicationProperties.getAddDayOfYearAttribute().getRegex()); - return processor; - } - - @Bean - @ConditionalOnProperty(prefix = "ningester.addTimeFromGranuleName", name = "enabled") - protected AddTimeFromGranuleName addTimeFromGranuleNameBean() { - - AddTimeFromGranuleName processor = new AddTimeFromGranuleName(applicationProperties.getAddTimeFromGranuleName().getRegex(), applicationProperties.getAddTimeFromGranuleName().getDateFormat()); - return processor; - } - - @Bean - @ConditionalOnProperty(prefix = "ningester.generateTileId", name = "enabled") - protected GenerateTileId generateTileIdBean() { - - GenerateTileId processor = new GenerateTileId(); - processor.setSalt(applicationProperties.getGenerateTileId().getSalt()); - return processor; - } - - @Bean - @ConditionalOnProperty(prefix = "ningester.pythonChainProcessor", name = "enabled") - protected PythonChainProcessor pythonChainProcessorBean(RestTemplate restTemplate) { - PythonChainProcessor processor = new PythonChainProcessor(restTemplate); - processor.setProcessorList(applicationProperties.getPythonChainProcessor().getProcessorList()); - processor.setUriPath(applicationProperties.getPythonChainProcessor().getUriPath()); - - return processor; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java deleted file mode 100644 index 7571173..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java +++ /dev/null @@ -1,122 +0,0 @@ -package gov.nasa.jpl.nexus.ningester.configuration; - -import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperties; -import gov.nasa.jpl.nexus.ningester.datatiler.FileSlicer; -import gov.nasa.jpl.nexus.ningester.datatiler.NetCDFItemReader; -import gov.nasa.jpl.nexus.ningester.processors.*; -import gov.nasa.jpl.nexus.ningester.writer.NexusWriter; -import org.apache.sdap.nexusproto.NexusTile; -import org.springframework.batch.core.Job; -import org.springframework.batch.core.Step; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; -import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; -import org.springframework.batch.core.configuration.annotation.JobScope; -import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; -import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.ItemStreamReader; -import org.springframework.batch.item.ItemWriter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; -import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.core.io.Resource; -import org.springframework.core.io.ResourceLoader; - -@Configuration -@EnableBatchProcessing -@Import(AppConfig.class) -public class BatchConfig { - - public static final String NINGESTER_JOB_NAME = "ningester"; - - @Autowired - protected JobBuilderFactory jobs; - - @Autowired - protected StepBuilderFactory steps; - - @Autowired - protected ApplicationProperties applicationProperties; - - @Autowired - protected ApplicationContext context; - - @Bean - public Job job(@Qualifier("ingestGranule") Step step1) { - return jobs.get(NINGESTER_JOB_NAME).start(step1).build(); - } - - @Bean - @JobScope - protected Resource granule(ResourceLoader resourceLoader, @Value("#{jobParameters['granule']}") String granuleLocation) { - return resourceLoader.getResource(granuleLocation); - } - - @Bean - @JobScope - protected ItemStreamReader<NexusTile> reader(FileSlicer fileSlicer, Resource granule) { - NetCDFItemReader reader = new NetCDFItemReader(fileSlicer); - reader.setResource(granule); - return reader; - } - - @Bean - @JobScope - protected ItemProcessor<NexusTile, NexusTile> processor() { - return new CompositeItemProcessor<>(applicationProperties.getTileProcessors()); - } - - @Bean - @JobScope - protected ItemWriter<NexusTile> writer(NexusWriter nexusWriter) { - return nexusWriter::saveToNexus; - } - - @Bean - @JobScope - protected Step ingestGranule(ItemStreamReader<NexusTile> reader, ItemProcessor<NexusTile, NexusTile> processor, ItemWriter<NexusTile> writer) { - return steps.get("ingestGranule") - .<NexusTile, NexusTile>chunk(10) - .reader(reader) - .processor(processor) - .writer(writer).build(); - } - - /* - * Item Processor beans defined below - */ - @Bean - @ConditionalOnBean(AddDatasetName.class) - protected ItemProcessor<NexusTile, NexusTile> addDatasetName(AddDatasetName addDatasetNameBean) { - return addDatasetNameBean::addDatasetName; - } - - @Bean - @ConditionalOnBean(AddDayOfYearAttribute.class) - protected ItemProcessor<NexusTile, NexusTile> addDayOfYearAttribute(AddDayOfYearAttribute addDayOfYearAttributeBean) { - return addDayOfYearAttributeBean::setDayOfYearFromGranuleName; - } - - @Bean - @ConditionalOnBean(AddTimeFromGranuleName.class) - protected ItemProcessor<NexusTile, NexusTile> addTimeFromGranuleName(AddTimeFromGranuleName addTimeFromGranuleNameBean) { - return addTimeFromGranuleNameBean::setTimeFromGranuleName; - } - - @Bean - @ConditionalOnBean(GenerateTileId.class) - protected ItemProcessor<NexusTile, NexusTile> generateTileId(GenerateTileId generateTileIdBean) { - return generateTileIdBean::addTileId; - } - - @Bean - @ConditionalOnBean(PythonChainProcessor.class) - protected ItemProcessor<NexusTile, NexusTile> pythonChainProcessor(PythonChainProcessor pythonChainProcessorBean) { - return pythonChainProcessorBean::nexusTileProcessor; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java deleted file mode 100644 index 7eee744..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java +++ /dev/null @@ -1,116 +0,0 @@ -package gov.nasa.jpl.nexus.ningester.configuration; - -import com.amazonaws.regions.Region; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; -import com.amazonaws.services.s3.AmazonS3Client; -import gov.nasa.jpl.nexus.ningester.configuration.properties.DatasourceProperties; -import gov.nasa.jpl.nexus.ningester.writer.*; -import org.apache.solr.client.solrj.SolrClient; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration; -import org.springframework.boot.autoconfigure.data.cassandra.CassandraDataAutoConfiguration; -import org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Profile; -import org.springframework.data.cassandra.core.CassandraTemplate; -import org.springframework.data.solr.core.SolrOperations; -import org.springframework.data.solr.core.SolrTemplate; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class DatasourceConfig { - - @Bean - @Profile("embedded") - public DataSource dataSource() { - EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder(); - return builder - .setType(EmbeddedDatabaseType.H2) - .build(); - } - - - @Configuration - @Profile("cassandra") - @Import({CassandraDataAutoConfiguration.class, CassandraAutoConfiguration.class}) - static class CassandraConfiguration { - - @Bean - public DataStore dataStore(CassandraTemplate cassandraTemplate) { - return new CassandraStore(cassandraTemplate); - } - } - - @Configuration - @Profile("dynamo") - static class DynamoConfiguration { - - @Autowired - private DatasourceProperties datasourceProperties; - - @Bean - public AmazonDynamoDB dynamoClient() { - AmazonDynamoDB dynamoClient = new AmazonDynamoDBClient(); - dynamoClient.setRegion(Region.getRegion(Regions.fromName(datasourceProperties.getDynamoStore().getRegion()))); - return dynamoClient; - } - - @Bean - public DataStore dataStore(AmazonDynamoDB dynamoClient) { - return new DynamoStore(dynamoClient, - datasourceProperties.getDynamoStore().getTableName(), - datasourceProperties.getDynamoStore().getPrimaryKey()); - } - } - - @Configuration - @Profile("s3") - static class S3Configuration { - @Autowired - private DatasourceProperties datasourceProperties; - - @Bean - public AmazonS3Client s3client() { - AmazonS3Client s3Client = new AmazonS3Client(); - s3Client.setRegion(Region.getRegion(Regions.fromName(datasourceProperties.getS3Store().getRegion()))); - return s3Client; - } - - @Bean - public DataStore dataStore(AmazonS3Client s3Client) { - return new S3Store(s3Client, datasourceProperties.getS3Store().getBucketName()); - } - } - - @Configuration - @Profile("solr") - @Import({SolrAutoConfiguration.class}) - static class SolrConfiguration { - - @Autowired - private DatasourceProperties datasourceProperties; - - @Bean - public SolrOperations solrTemplate(SolrClient solrClient) { - return new SolrTemplate(solrClient); - } - - - @Bean - public MetadataStore metadataStore(SolrOperations solrTemplate) { - SolrStore solrStore = new SolrStore(solrTemplate); - solrStore.setCollection(datasourceProperties.getSolrStore().getCollection()); - solrStore.setCommitWithin(datasourceProperties.getSolrStore().getCommitWithin()); - solrStore.setGeoPrecision(datasourceProperties.getSolrStore().getGeoPrecision()); - - return solrStore; - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java deleted file mode 100644 index ced72ee..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java +++ /dev/null @@ -1,78 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.configuration.properties; - -import gov.nasa.jpl.nexus.ningester.datatiler.properties.SliceFileByDimension; -import gov.nasa.jpl.nexus.ningester.datatiler.properties.SliceFileByTilesDesired; -import gov.nasa.jpl.nexus.ningester.processors.properties.*; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.NestedConfigurationProperty; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.List; - -@ConfigurationProperties("ningester") -@Component -public class ApplicationProperties { - - @NestedConfigurationProperty - private final SliceFileByDimension sliceFileByDimension = new SliceFileByDimension(); - @NestedConfigurationProperty - private final SliceFileByTilesDesired sliceFileByTilesDesired = new SliceFileByTilesDesired(); - @NestedConfigurationProperty - private final AddDayOfYearAttribute addDayOfYearAttribute = new AddDayOfYearAttribute(); - @NestedConfigurationProperty - private final AddTimeFromGranuleName addTimeFromGranuleName = new AddTimeFromGranuleName(); - @NestedConfigurationProperty - private final GenerateTileId generateTileId = new GenerateTileId(); - @NestedConfigurationProperty - private final PythonChainProcessor pythonChainProcessor = new PythonChainProcessor(); - @NestedConfigurationProperty - private final AddDatasetName addDatasetName = new AddDatasetName(); - private String tileSlicer; - private List<String> tileProcessors = new ArrayList<>(); - - public PythonChainProcessor getPythonChainProcessor() { - return pythonChainProcessor; - } - - public List<String> getTileProcessors() { - return tileProcessors; - } - - public AddDayOfYearAttribute getAddDayOfYearAttribute() { - return addDayOfYearAttribute; - } - - public AddTimeFromGranuleName getAddTimeFromGranuleName() { - return addTimeFromGranuleName; - } - - public GenerateTileId getGenerateTileId() { - return generateTileId; - } - - public AddDatasetName getAddDatasetName() { - return addDatasetName; - } - - public String getTileSlicer() { - return tileSlicer; - } - - public void setTileSlicer(String tileSlicer) { - this.tileSlicer = tileSlicer; - } - - public SliceFileByTilesDesired getSliceFileByTilesDesired() { - return sliceFileByTilesDesired; - } - - public SliceFileByDimension getSliceFileByDimension() { - return sliceFileByDimension; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java deleted file mode 100644 index 9f5c5e7..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - ***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.configuration.properties; - -import gov.nasa.jpl.nexus.ningester.writer.properties.CassandraStore; -import gov.nasa.jpl.nexus.ningester.writer.properties.DynamoStore; -import gov.nasa.jpl.nexus.ningester.writer.properties.S3Store; -import gov.nasa.jpl.nexus.ningester.writer.properties.SolrStore; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.NestedConfigurationProperty; -import org.springframework.stereotype.Component; - -@ConfigurationProperties("datasource") -@Component -public class DatasourceProperties { - - @NestedConfigurationProperty - private final CassandraStore cassandraStore = new CassandraStore(); - - @NestedConfigurationProperty - private final DynamoStore dynamoStore = new DynamoStore(); - - @NestedConfigurationProperty - private final S3Store s3Store = new S3Store(); - - @NestedConfigurationProperty - private final SolrStore solrStore = new SolrStore(); - - public DynamoStore getDynamoStore() { - return dynamoStore; - } - - public S3Store getS3Store() { - return s3Store; - } - - public SolrStore getSolrStore() { - return solrStore; - } - - public CassandraStore getCassandraStore() { - return cassandraStore; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.java deleted file mode 100644 index 3258119..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.java +++ /dev/null @@ -1,16 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.datatiler; - -import java.io.File; -import java.io.IOException; -import java.util.List; - -public interface FileSlicer { - - List<String> generateSlices(File inputfile) throws IOException; - -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java deleted file mode 100644 index f5646d1..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java +++ /dev/null @@ -1,124 +0,0 @@ -package gov.nasa.jpl.nexus.ningester.datatiler; - -import org.apache.sdap.nexusproto.NexusTile; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.batch.item.ExecutionContext; -import org.springframework.batch.item.ItemStreamException; -import org.springframework.batch.item.UnexpectedInputException; -import org.springframework.batch.item.file.ResourceAwareItemReaderItemStream; -import org.springframework.core.io.Resource; -import ucar.nc2.dataset.NetcdfDataset; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -public class NetCDFItemReader implements ResourceAwareItemReaderItemStream<NexusTile> { - - static final String CURRENT_TILE_SPEC_INDEX_KEY = "current.tile.spec.index"; - private static final Logger log = LoggerFactory.getLogger(NetCDFItemReader.class); - private List<String> tileSpecList; - private Integer currentTileSpecIndex; - - private Resource netCDFResource; - private NetcdfDataset ds; - private FileSlicer fileSlicer; - - /** - * Constructor - * - * @param fileSlicer Object responsible for slicing the NetCDF file into tiles. - */ - public NetCDFItemReader(FileSlicer fileSlicer) { - this.fileSlicer = fileSlicer; - } - - @Override - public NexusTile read() { - if (this.currentTileSpecIndex == this.tileSpecList.size()) { - //End of stream - return null; - } - String currentSpec = this.tileSpecList.get(this.currentTileSpecIndex); - - URL netCDFUrl = null; - try { - netCDFUrl = this.netCDFResource.getURL(); - } catch (IOException e) { - throw new UnexpectedInputException("Generic IOException", e); - } - - NexusTile.Builder nexusTileBuilder = NexusTile.newBuilder(); - nexusTileBuilder.getSummaryBuilder() - .setSectionSpec(currentSpec) - .setGranule(netCDFUrl.toString()); - - this.currentTileSpecIndex++; - return nexusTileBuilder.build(); - } - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - - File netCDFFile = null; - try { - netCDFFile = this.netCDFResource.getFile(); - } catch (IOException e) { - throw new ItemStreamException(e); - } - - //Every time we open the file we generate the tile specs according to the given file slicer - try { - this.tileSpecList = fileSlicer.generateSlices(netCDFFile); - } catch (IOException e) { - throw new ItemStreamException(e); - } - log.debug("Generated tile specifications for {}\nINDEX\tTILE SPECIFICATION\n{}", netCDFFile.getName(), - IntStream.range(0, this.tileSpecList.size()) - .mapToObj(i -> i + "\t" + this.tileSpecList.get(i)) - .collect(Collectors.joining("\n"))); - - if (!executionContext.containsKey(CURRENT_TILE_SPEC_INDEX_KEY)) { - //Start at index 0 - this.currentTileSpecIndex = 0; - executionContext.putInt(CURRENT_TILE_SPEC_INDEX_KEY, this.currentTileSpecIndex); - } else { - //Start at index location from context - this.currentTileSpecIndex = executionContext.getInt(CURRENT_TILE_SPEC_INDEX_KEY); - } - - //Open the resource - try { - this.ds = NetcdfDataset.openDataset(netCDFFile.getAbsolutePath()); - } catch (IOException e) { - throw new ItemStreamException(e); - } - - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException { - - executionContext.putInt(CURRENT_TILE_SPEC_INDEX_KEY, this.currentTileSpecIndex); - } - - @Override - public void close() throws ItemStreamException { - - try { - this.ds.close(); - } catch (IOException e) { - throw new ItemStreamException(e); - } - - } - - @Override - public void setResource(Resource resource) { - this.netCDFResource = resource; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java deleted file mode 100644 index d431397..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * **************************************************************************** - * Copyright (c) 2016 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - * ****************************************************************************/ -package gov.nasa.jpl.nexus.ningester.datatiler; - -import com.google.common.base.Strings; -import com.google.common.collect.Sets; -import ucar.nc2.Dimension; -import ucar.nc2.Variable; -import ucar.nc2.dataset.NetcdfDataset; - -import java.io.File; -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -public class SliceFileByDimension implements FileSlicer { - - private String sliceByDimension; - private List<String> dimensions; - private String dimensionNamePrefix; - - public void setDimensions(List<String> dims) { - this.dimensions = dims; - } - - public void setSliceByDimension(String sliceBy) { - this.sliceByDimension = sliceBy; - } - - public void setDimensionNamePrefix(String dimensionNamePrefix) { - this.dimensionNamePrefix = dimensionNamePrefix; - } - - public List<String> generateSlices(File inputfile) throws IOException { - - boolean isInteger = false; - try { - Integer.parseInt(this.sliceByDimension); - isInteger = true; - } catch (NumberFormatException e) { - //ignore - } - return (isInteger) ? indexedDimensionSlicing(inputfile) : namedDimensionSlicing(inputfile); - } - - List<String> indexedDimensionSlicing(File inputfile) throws IOException { - - // This is sort of a hack to help the python netcdf library. When you try to get dimensions by name and they are unnamed, the - // python library uses 'phony_dim_' then the index of the dimension as the dimension name. Weird, I know. - if (Strings.isNullOrEmpty(this.dimensionNamePrefix)) { - this.dimensionNamePrefix = "phony_dim_"; - } - Map<String, Integer> dimensionNameToLength; - try (NetcdfDataset ds = NetcdfDataset.openDataset(inputfile.getAbsolutePath())) { - - - // Because this is indexed-based dimension slicing, the dimensions are assumed to be unlimited with no names (ie. ds.dimensions == []) - // Therefore, we need to find a 'representative' variable with dimensions that we can inspect and work with - // 'lat' and 'lon' are common variable names in the datasets we work with. So try to find one of those first - // Otherwise, just find the first variable that has the same number of dimensions as was given in this.dimensions - List<String> commonVariableNames = Arrays.asList("lat", "latitude", "lon", "longitude"); - Optional<Variable> var = ds.getVariables().stream() - .filter(variable -> commonVariableNames.contains(variable.getShortName().toLowerCase()) - || variable.getDimensions().size() == this.dimensions.size()) - .findFirst(); - - assert var.isPresent() : "Could not find a variable in " + inputfile.getName() + " with " + dimensions.size() + " dimension(s)."; - - dimensionNameToLength = IntStream.range(0, this.dimensions.size()).boxed() - .collect(Collectors.toMap(dimIndex -> this.dimensionNamePrefix + dimIndex, dimIndex -> var.get().getDimension(dimIndex).getLength())); - } - - return generateTileBoundrySlices(this.dimensionNamePrefix + this.sliceByDimension, dimensionNameToLength); - - } - - List<String> namedDimensionSlicing(File inputfile) throws IOException { - Map<String, Integer> dimensionNameToLength; - try (NetcdfDataset ds = NetcdfDataset.openDataset(inputfile.getAbsolutePath())) { - - dimensionNameToLength = ds.getDimensions().stream() - .filter(dimension -> this.dimensions.contains(dimension.getShortName())) - .collect(Collectors.toMap(Dimension::getShortName, Dimension::getLength)); - } - - return generateTileBoundrySlices(this.sliceByDimension, dimensionNameToLength); - } - - List<String> generateTileBoundrySlices(String sliceByDimension, Map<String, Integer> dimensionNameToLength) { - - List<Set<String>> dimensionBounds = dimensionNameToLength.entrySet().stream() - .map(stringIntegerEntry -> { - String dimensionName = stringIntegerEntry.getKey(); - Integer lengthOfDimension = stringIntegerEntry.getValue(); - Integer stepSize = (dimensionName.equals(sliceByDimension)) ? 1 : lengthOfDimension; - - Set<String> bounds = new LinkedHashSet<>(); - for (int i = 0; i < lengthOfDimension; i += stepSize) { - bounds.add( - dimensionName + ":" + - i + ":" + - (i + stepSize >= lengthOfDimension ? lengthOfDimension : i + stepSize)); - } - return bounds; - }).collect(Collectors.toList()); - - return Sets.cartesianProduct(dimensionBounds) - .stream() - .map(tileSpecAsList -> tileSpecAsList.stream().collect(Collectors.joining(","))) - .collect(Collectors.toList()); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java deleted file mode 100644 index 05a2b45..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java +++ /dev/null @@ -1,104 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ -package gov.nasa.jpl.nexus.ningester.datatiler; - -import com.google.common.collect.Sets; -import ucar.nc2.Dimension; -import ucar.nc2.dataset.NetcdfDataset; - -import java.io.File; -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -public class SliceFileByTilesDesired implements FileSlicer { - - private Integer tilesDesired; - private List<String> dimensions; - private String timeDimension; - - public void setTilesDesired(Integer desired) { - this.tilesDesired = desired; - } - - public void setDimensions(List<String> dims) { - this.dimensions = dims; - } - - @Override - public List<String> generateSlices(File inputfile) throws IOException { - - Integer timeLen = 0; - Map<String, Integer> dimensionNameToLength; - try (NetcdfDataset ds = NetcdfDataset.openDataset(inputfile.getAbsolutePath())) { - - - dimensionNameToLength = ds.getDimensions().stream() - .filter(dimension -> this.dimensions.contains(dimension.getShortName())) - .collect(Collectors.toMap(Dimension::getShortName, Dimension::getLength, - (v1, v2) -> { - throw new RuntimeException(String.format("Duplicate key for values %s and %s", v1, v2)); - }, - TreeMap::new)); - - if (this.timeDimension != null) { - timeLen = ds.getDimensions().stream() - .filter(dimension -> this.timeDimension.equals(dimension.getShortName())) - .map(Dimension::getLength) - .collect(Collectors.toList()).get(0); - } - } - - return addTimeDimension(generateChunkBoundrySlices(tilesDesired, dimensionNameToLength), timeLen); - - } - - List<String> generateChunkBoundrySlices(Integer tilesDesired, Map<String, Integer> dimensionNameToLength) { - - List<Set<String>> dimensionBounds = dimensionNameToLength.entrySet().stream() - .map(stringIntegerEntry -> { - String dimensionName = stringIntegerEntry.getKey(); - Integer lengthOfDimension = stringIntegerEntry.getValue(); - Integer stepSize = calculateStepSize(stringIntegerEntry.getValue(), tilesDesired, dimensionNameToLength.size()); - Set<String> bounds = new LinkedHashSet<>(); - for (int i = 0; i < lengthOfDimension; i += stepSize) { - bounds.add( - dimensionName + ":" + - i + ":" + - (i + stepSize >= lengthOfDimension ? lengthOfDimension : i + stepSize)); - } - return bounds; - }).collect(Collectors.toList()); - - return Sets.cartesianProduct(dimensionBounds) - .stream() - .map(tileSpecAsList -> tileSpecAsList.stream().collect(Collectors.joining(","))) - .collect(Collectors.toList()); - - } - - List<String> addTimeDimension(List<String> specs, Integer timeLen) { - - if (timeLen > 0) { - return specs.stream().map(sectionSpec -> - IntStream.range(0, timeLen) - .mapToObj(timeIndex -> this.timeDimension + ":" + timeIndex + ":" + (timeIndex + 1) + "," + sectionSpec) - .collect(Collectors.toList())) - .flatMap(List::stream) - .collect(Collectors.toList()); - } else { - return specs; - } - } - - private Integer calculateStepSize(Integer lengthOfDimension, Integer chunksDesired, Integer numberOfDimensions) { - return new Double(Math.floor(lengthOfDimension / (Math.pow(chunksDesired, (1.0 / numberOfDimensions))))).intValue(); - } - - public void setTimeDimension(String timeDimension) { - this.timeDimension = timeDimension; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByDimension.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByDimension.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByDimension.java deleted file mode 100644 index 0482e55..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByDimension.java +++ /dev/null @@ -1,44 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.datatiler.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -import java.util.List; - -@ConfigurationProperties -@Component("sliceFileByDimensionProperties") -public class SliceFileByDimension { - - private String sliceByDimension; - private List<String> dimensions; - private String dimensionNamePrefix = ""; - - public String getSliceByDimension() { - return sliceByDimension; - } - - public void setSliceByDimension(String sliceByDimension) { - this.sliceByDimension = sliceByDimension; - } - - public List<String> getDimensions() { - return dimensions; - } - - public void setDimensions(List<String> dimensions) { - this.dimensions = dimensions; - } - - public String getDimensionNamePrefix() { - return dimensionNamePrefix; - } - - public void setDimensionNamePrefix(String dimensionNamePrefix) { - this.dimensionNamePrefix = dimensionNamePrefix; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByTilesDesired.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByTilesDesired.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByTilesDesired.java deleted file mode 100644 index cc7933b..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByTilesDesired.java +++ /dev/null @@ -1,44 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.datatiler.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.List; - -@ConfigurationProperties -@Component("sliceFileByTilesDesiredProperties") -public class SliceFileByTilesDesired { - private Integer tilesDesired; - private List<String> dimensions = new ArrayList<>(); - private String timeDimension; - - public Integer getTilesDesired() { - return tilesDesired; - } - - public void setTilesDesired(Integer tilesDesired) { - this.tilesDesired = tilesDesired; - } - - public List<String> getDimensions() { - return dimensions; - } - - public void setDimensions(List<String> dimensions) { - this.dimensions = dimensions; - } - - public String getTimeDimension() { - return timeDimension; - } - - public void setTimeDimension(String timeDimension) { - this.timeDimension = timeDimension; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/http/NexusTileConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/http/NexusTileConverter.java b/src/main/java/gov/nasa/jpl/nexus/ningester/http/NexusTileConverter.java deleted file mode 100644 index 8305e66..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/http/NexusTileConverter.java +++ /dev/null @@ -1,40 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.http; - -import org.apache.sdap.nexusproto.NexusTile; -import org.springframework.http.HttpInputMessage; -import org.springframework.http.HttpOutputMessage; -import org.springframework.http.converter.AbstractHttpMessageConverter; -import org.springframework.http.converter.HttpMessageNotReadableException; -import org.springframework.http.converter.HttpMessageNotWritableException; - -import java.io.IOException; -import java.io.OutputStream; - -public class NexusTileConverter extends AbstractHttpMessageConverter<NexusTile> { - - @Override - protected boolean supports(Class<?> clazz) { - return NexusTile.class.isAssignableFrom(clazz); - } - - @Override - protected NexusTile readInternal(Class<? extends NexusTile> clazz, HttpInputMessage inputMessage) throws IOException, HttpMessageNotReadableException { - - return NexusTile.parseFrom(inputMessage.getBody()); - } - - @Override - protected void writeInternal(NexusTile nexusTile, HttpOutputMessage outputMessage) throws IOException, HttpMessageNotWritableException { - try { - OutputStream outputStream = outputMessage.getBody(); - nexusTile.writeTo(outputStream); - outputStream.close(); - } catch (Exception ignored) { - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDatasetName.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDatasetName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDatasetName.java deleted file mode 100644 index b08757a..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDatasetName.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - ****************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors; - -import org.apache.sdap.nexusproto.NexusTile; - -public class AddDatasetName { - - String datasetName; - - public AddDatasetName(String datasetName) { - this.datasetName = datasetName; - } - - public NexusTile addDatasetName(NexusTile inputTile) { - - NexusTile.Builder outTileBuilder = NexusTile.newBuilder().mergeFrom(inputTile); - - outTileBuilder.getSummaryBuilder().setDatasetName(datasetName); - - return outTileBuilder.build(); - - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDayOfYearAttribute.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDayOfYearAttribute.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDayOfYearAttribute.java deleted file mode 100644 index c22cb34..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDayOfYearAttribute.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - ***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors; - -import org.apache.sdap.nexusproto.Attribute; -import org.apache.sdap.nexusproto.NexusTile; -import org.apache.sdap.nexusproto.TileSummary; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class AddDayOfYearAttribute { - - private Pattern regex; - - /** - * Reuqires regex that defines EXACTLY one capturing group that contains the day of year when matched with the - * granule name. - * - * @param regex Regex used to match against the granule name. Must define exactly one capturing group that captures - * the day of year. - */ - public AddDayOfYearAttribute(String regex) { - this.regex = Pattern.compile(regex); - } - - /** - * Uses regex to extract a match from the granule name that contains the day of year. - * - * @param nexusTile The tile to process - * @return The processed tile - */ - public NexusTile setDayOfYearFromGranuleName(NexusTile nexusTile) { - - String granuleName = nexusTile.getSummary().getGranule(); - Matcher granuleNameMatcher = this.regex.matcher(granuleName); - Boolean granuleNameMatched = granuleNameMatcher.find(); - - if (!granuleNameMatched) { - throw new RuntimeException("regex did not match granuleName."); - } - - if (granuleNameMatcher.groupCount() != 1) { - throw new RuntimeException("regex does not have exactly one capturing group."); - } - - if (granuleNameMatcher.group(1).length() <= 0) { - throw new RuntimeException("group does not contain match."); - } - - - String dayOfYear = granuleNameMatcher.group(1); - NexusTile.Builder newTileBuilder = NexusTile.newBuilder().mergeFrom(nexusTile); - TileSummary.Builder newTileSummaryBuilder = newTileBuilder.getSummaryBuilder(); - newTileSummaryBuilder.addGlobalAttributes( - Attribute.newBuilder() - .setName("day_of_year_i") - .addValues(dayOfYear) - .build() - ); - newTileBuilder.setSummary(newTileSummaryBuilder); - - return newTileBuilder.build(); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeFromGranuleName.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeFromGranuleName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeFromGranuleName.java deleted file mode 100644 index 2886b93..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeFromGranuleName.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - ***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ -package gov.nasa.jpl.nexus.ningester.processors; - -import org.apache.sdap.nexusproto.NexusTile; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Locale; -import java.util.TimeZone; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - - -public class AddTimeFromGranuleName { - - - private Pattern regex; - private SimpleDateFormat dateFormat; - - public AddTimeFromGranuleName(String regex, String dateFormat) { - this.regex = Pattern.compile(regex); - this.dateFormat = new SimpleDateFormat(dateFormat, Locale.US); - this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - } - - public NexusTile setTimeFromGranuleName(NexusTile inputTile) { - - NexusTile.Builder outTileBuilder = NexusTile.newBuilder().mergeFrom(inputTile); - - switch (inputTile.getTile().getTileTypeCase()) { - case GRID_TILE: - - String granuleName = inputTile.getSummary().getGranule(); - Matcher granuleNameMatcher = this.regex.matcher(granuleName); - Boolean granuleNameMatched = granuleNameMatcher.find(); - - if (!granuleNameMatched) { - throw new RuntimeException("regex did not match granuleName."); - } - - if (granuleNameMatcher.groupCount() != 1) { - throw new RuntimeException("regex does not have exactly one capturing group."); - } - - if (granuleNameMatcher.group(1).length() <= 0) { - throw new RuntimeException("group does not contain match."); - } - - String dateTimeString = granuleNameMatcher.group(1); - Date dateTime = null; - try { - dateTime = dateFormat.parse(dateTimeString); - } catch (ParseException e) { - throw new RuntimeException(e); - } - - Long secondsSinceEpoch = (dateTime.getTime() / 1000); - - outTileBuilder.getTileBuilder().getGridTileBuilder().setTime(secondsSinceEpoch); - outTileBuilder.getSummaryBuilder().getStatsBuilder().setMinTime(secondsSinceEpoch); - outTileBuilder.getSummaryBuilder().getStatsBuilder().setMaxTime(secondsSinceEpoch); - break; - default: - throw new UnsupportedOperationException("Can only handle GridTile at this time."); - } - - return outTileBuilder.build(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/processors/CompositeItemProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/CompositeItemProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/CompositeItemProcessor.java deleted file mode 100644 index af91307..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/CompositeItemProcessor.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - ***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors; - -import org.springframework.batch.item.ItemProcessor; -import org.springframework.beans.BeansException; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; - -import java.util.ArrayList; -import java.util.List; - -public class CompositeItemProcessor<I, O> extends org.springframework.batch.item.support.CompositeItemProcessor<I, O> implements ApplicationContextAware { - - private ApplicationContext applicationContext; - - private List<String> processorBeanNames; - - public CompositeItemProcessor(List<String> processorBeanNames) { - this.processorBeanNames = processorBeanNames; - } - - @Override - @SuppressWarnings("unchecked") - public void afterPropertiesSet() { - List<ItemProcessor<I, O>> delegates = new ArrayList<>(); - for (String processorBeanName : processorBeanNames) { - delegates.add(applicationContext.getBean(processorBeanName, ItemProcessor.class)); - } - - setDelegates(delegates); - } - - @Override - public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - this.applicationContext = applicationContext; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java deleted file mode 100644 index dd1913b..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - ***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors; - -import com.google.common.io.Files; -import org.apache.sdap.nexusproto.NexusTile; - -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.UUID; - -public class GenerateTileId { - - private String salt = ""; - - public void setSalt(String salt) { - this.salt = salt; - } - - public NexusTile addTileId(NexusTile inputTile) { - - NexusTile.Builder outTileBuilder = NexusTile.newBuilder().mergeFrom(inputTile); - String granuleFileName = inputTile.getSummary().getGranule(); - Path granulePath = null; - try { - granulePath = Paths.get(new URI(granuleFileName)); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - String granuleName = Files.getNameWithoutExtension(granulePath.getFileName().toString()); - String spec = inputTile.getSummary().getSectionSpec(); - - String tileId = UUID.nameUUIDFromBytes((granuleName + spec + salt).getBytes()).toString(); - outTileBuilder.getSummaryBuilder().setTileId(tileId); - outTileBuilder.getTileBuilder().setTileId(tileId); - - return outTileBuilder.build(); - } - - -} - http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java deleted file mode 100644 index a54f49e..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - ***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import gov.nasa.jpl.nexus.ningester.processors.properties.PythonProcessorModule; -import org.apache.sdap.nexusproto.NexusTile; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.MediaType; -import org.springframework.web.client.RestTemplate; - -import java.util.Collections; -import java.util.List; - -public class PythonChainProcessor { - - private RestTemplate restTemplate; - - private List<PythonProcessorModule> processorList; - - private String uriPath; - - public PythonChainProcessor(RestTemplate restTemplate) { - this.restTemplate = restTemplate; - } - - public NexusTile nexusTileProcessor(NexusTile nexusTile) { - - HttpHeaders headers = new HttpHeaders(); - headers.setAccept(Collections.singletonList(MediaType.APPLICATION_OCTET_STREAM)); - headers.setContentType(MediaType.APPLICATION_JSON); - - PythonChainProcessorRequest chainProcessorRequest = new PythonChainProcessorRequest(); - chainProcessorRequest.setProcessorList(processorList); - try { - chainProcessorRequest.setNexusTile(JsonFormat.printer().print(nexusTile)); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - - HttpEntity<PythonChainProcessorRequest> requestEntity = new HttpEntity<>(chainProcessorRequest, headers); - - return restTemplate.exchange( - uriPath, - HttpMethod.POST, - requestEntity, - NexusTile.class).getBody(); - } - - public void setProcessorList(List<PythonProcessorModule> processorList) { - this.processorList = processorList; - } - - public void setUriPath(String uriPath) { - this.uriPath = uriPath; - } - - public class PythonChainProcessorRequest { - - @JsonProperty("processor_list") - private List<PythonProcessorModule> processorList; - - @JsonProperty("input_data") - private String nexusTile; - - public List<PythonProcessorModule> getProcessorList() { - return processorList; - } - - public void setProcessorList(List<PythonProcessorModule> processorList) { - this.processorList = processorList; - } - - public String getNexusTile() { - return nexusTile; - } - - public void setNexusTile(String nexusTile) { - this.nexusTile = nexusTile; - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDatasetName.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDatasetName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDatasetName.java deleted file mode 100644 index 7bd9af0..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDatasetName.java +++ /dev/null @@ -1,24 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@ConfigurationProperties -@Component("addDatasetNameProperties") -public class AddDatasetName { - - private String datasetName; - - public String getDatasetName() { - return datasetName; - } - - public void setDatasetName(String datasetName) { - this.datasetName = datasetName; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDayOfYearAttribute.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDayOfYearAttribute.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDayOfYearAttribute.java deleted file mode 100644 index 33d8b24..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDayOfYearAttribute.java +++ /dev/null @@ -1,25 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@ConfigurationProperties -@Component("addDayOfYearAttributeProperties") -public class AddDayOfYearAttribute { - - private String regex; - - - public String getRegex() { - return regex; - } - - public void setRegex(String regex) { - this.regex = regex; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddTimeFromGranuleName.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddTimeFromGranuleName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddTimeFromGranuleName.java deleted file mode 100644 index 1042d6a..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddTimeFromGranuleName.java +++ /dev/null @@ -1,33 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@ConfigurationProperties -@Component("addTimeFromGranuleNameProperties") -public class AddTimeFromGranuleName { - - private String regex; - private String dateFormat; - - public String getRegex() { - return regex; - } - - public void setRegex(String regex) { - this.regex = regex; - } - - public String getDateFormat() { - return dateFormat; - } - - public void setDateFormat(String dateFormat) { - this.dateFormat = dateFormat; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/GenerateTileId.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/GenerateTileId.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/GenerateTileId.java deleted file mode 100644 index 60cf75c..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/GenerateTileId.java +++ /dev/null @@ -1,24 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@ConfigurationProperties -@Component("generateTileIdProperties") -public class GenerateTileId { - - private String salt = ""; - - public String getSalt() { - return salt; - } - - public void setSalt(String salt) { - this.salt = salt; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonChainProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonChainProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonChainProcessor.java deleted file mode 100644 index ac86d99..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonChainProcessor.java +++ /dev/null @@ -1,42 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -import java.net.URL; -import java.util.ArrayList; -import java.util.List; - -@ConfigurationProperties -@Component("pythonChainProcessorProperties") -public class PythonChainProcessor { - private URL baseUrl; - private String uriPath; - - private List<PythonProcessorModule> processorList = new ArrayList<>(); - - public String getUriPath() { - return uriPath; - } - - public void setUriPath(String uriPath) { - this.uriPath = uriPath; - } - - public List<PythonProcessorModule> getProcessorList() { - return processorList; - } - - public URL getBaseUrl() { - return baseUrl; - } - - public void setBaseUrl(URL baseUrl) { - this.baseUrl = baseUrl; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonProcessorModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonProcessorModule.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonProcessorModule.java deleted file mode 100644 index e48f0d5..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonProcessorModule.java +++ /dev/null @@ -1,27 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors.properties; - -import java.util.HashMap; -import java.util.Map; - -public class PythonProcessorModule { - - private String name; - private Map<String, String> config = new HashMap<>(); - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public Map<String, String> getConfig() { - return config; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java deleted file mode 100644 index c466d7b..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - ***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.writer; - -import org.apache.sdap.nexusproto.NexusTile; -import org.apache.sdap.nexusproto.TileData; -import org.springframework.data.cassandra.core.CassandraOperations; - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; -import java.util.stream.Collectors; - -public class CassandraStore implements DataStore { - - private CassandraOperations cassandraTemplate; - - //TODO This will be refactored at some point to be dynamic per-message. Or maybe per-group. - private String tableName = "sea_surface_temp"; - - public CassandraStore(CassandraOperations cassandraTemplate) { - this.cassandraTemplate = cassandraTemplate; - } - - @Override - public void saveData(List<? extends NexusTile> nexusTiles) { - - String query = "insert into " + this.tableName + " (tile_id, tile_blob) VALUES (?, ?)"; - cassandraTemplate.ingest(query, nexusTiles.stream() - .map(nexusTile -> getCassandraRowFromTileData(nexusTile.getTile())) - .collect(Collectors.toList())); - } - - private List<Object> getCassandraRowFromTileData(TileData tile) { - - UUID tileId = UUID.fromString(tile.getTileId()); - return Arrays.asList(tileId, ByteBuffer.wrap(tile.toByteArray())); - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStore.java deleted file mode 100644 index d1fd52a..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStore.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * **************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.writer; - -import org.apache.sdap.nexusproto.NexusTile; - -import java.util.List; - -public interface DataStore { - - void saveData(List<? extends NexusTile> nexusTiles); -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStoreException.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStoreException.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStoreException.java deleted file mode 100644 index 9ba838e..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStoreException.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - ***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ -package gov.nasa.jpl.nexus.ningester.writer; - -public class DataStoreException extends RuntimeException { - - public DataStoreException() { - } - - public DataStoreException(String message) { - super(message); - } - - public DataStoreException(String message, Throwable cause) { - super(message, cause); - } - - public DataStoreException(Throwable cause) { - super(cause); - } - - public DataStoreException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java deleted file mode 100644 index b0f6549..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - ***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ -package gov.nasa.jpl.nexus.ningester.writer; - -import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; -import com.amazonaws.services.dynamodbv2.document.DynamoDB; -import com.amazonaws.services.dynamodbv2.document.Item; -import com.amazonaws.services.dynamodbv2.document.Table; -import org.apache.sdap.nexusproto.NexusTile; - -import java.util.List; - -/** - * Created by djsilvan on 6/26/17. - */ -public class DynamoStore implements DataStore { - - private DynamoDB dynamoDB; - private String tableName; - private String primaryKey; - - public DynamoStore(AmazonDynamoDB dynamoClient, String tableName, String primaryKey) { - dynamoDB = new DynamoDB(dynamoClient); - this.tableName = tableName; - this.primaryKey = primaryKey; - } - - public void saveData(List<? extends NexusTile> nexusTiles) { - - Table table = dynamoDB.getTable(tableName); - - for (NexusTile tile : nexusTiles) { - String tileId = getTileId(tile); - byte[] tileData = getTileData(tile); - - try { - table.putItem(new Item().withPrimaryKey(primaryKey, tileId).withBinary("data", tileData)); - } catch (Exception e) { - throw new DataStoreException("Unable to add item: " + tileId, e); - } - } - } - - private String getTileId(NexusTile tile) { - return tile.getTile().getTileId(); - } - - private byte[] getTileData(NexusTile tile) { - return tile.getTile().toByteArray(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/writer/MetadataStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/MetadataStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/MetadataStore.java deleted file mode 100644 index 69a0937..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/MetadataStore.java +++ /dev/null @@ -1,17 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.writer; - -import org.apache.sdap.nexusproto.NexusTile; - -import java.util.List; - -public interface MetadataStore { - - void saveMetadata(List<? extends NexusTile> nexusTiles); - - void deleteMetadata(List<? extends NexusTile> nexusTiles); -}
