added other datastores and their properties
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/a52c618e Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/a52c618e Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/a52c618e Branch: refs/heads/master Commit: a52c618eec170634f4b631ac87c2d5afb13de77e Parents: 53af516 Author: Frank Greguska <[email protected]> Authored: Mon Jan 8 18:05:39 2018 -0800 Committer: Frank Greguska <[email protected]> Committed: Mon Jan 8 18:05:39 2018 -0800 ---------------------------------------------------------------------- build.gradle | 9 + .../configuration/DatasourceConfig.java | 167 ++++++++++++++++++- .../properties/AddDatasetName.java | 24 --- .../properties/AddDayOfYearAttribute.java | 25 --- .../properties/AddTimeFromGranuleName.java | 33 ---- .../properties/ApplicationProperties.java | 3 + .../properties/DatasourceProperties.java | 47 ++++++ .../properties/GenerateTileId.java | 24 --- .../properties/PythonChainProcessor.java | 42 ----- .../properties/PythonProcessorModule.java | 27 --- .../properties/SliceFileByDimension.java | 44 ----- .../properties/SliceFileByTilesDesired.java | 44 ----- .../properties/SliceFileByDimension.java | 44 +++++ .../properties/SliceFileByTilesDesired.java | 44 +++++ .../processors/PythonChainProcessor.java | 2 +- .../processors/properties/AddDatasetName.java | 24 +++ .../properties/AddDayOfYearAttribute.java | 25 +++ .../properties/AddTimeFromGranuleName.java | 33 ++++ .../processors/properties/GenerateTileId.java | 24 +++ .../properties/PythonChainProcessor.java | 42 +++++ .../properties/PythonProcessorModule.java | 27 +++ .../ningester/writer/DataStoreException.java | 28 ++++ .../jpl/nexus/ningester/writer/DynamoStore.java | 54 ++++++ .../jpl/nexus/ningester/writer/S3Store.java | 62 +++++++ .../jpl/nexus/ningester/writer/SolrStore.java | 3 +- .../writer/properties/CassandraStore.java | 42 +++++ .../writer/properties/DynamoStore.java | 44 +++++ .../ningester/writer/properties/S3Store.java | 34 ++++ .../ningester/writer/properties/SolrStore.java | 44 +++++ src/main/resources/application.properties | 0 src/main/resources/application.yml | 5 + 31 files changed, 797 insertions(+), 273 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index ebabedf..7ceb035 100644 --- a/build.gradle +++ b/build.gradle @@ -59,6 +59,12 @@ sourceSets{ } } +dependencyManagement { + imports { + mavenBom 'com.amazonaws:aws-java-sdk-bom:1.10.77+' + } +} + dependencies { optional("org.springframework.boot:spring-boot-configuration-processor") @@ -74,6 +80,9 @@ dependencies { compile("edu.ucar:cdm:${netcdfJavaVersion}") compile("com.google.guava:guava") + compile("com.amazonaws:aws-java-sdk-s3") + compile("com.amazonaws:aws-java-sdk-dynamodb") + testCompile('org.springframework.boot:spring-boot-starter-test') testCompile('org.springframework.batch:spring-batch-test') } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java index 4868673..de4fef2 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java @@ -1,8 +1,32 @@ package gov.nasa.jpl.nexus.ningester.configuration; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; +import com.amazonaws.services.s3.AmazonS3Client; +import gov.nasa.jpl.nexus.ningester.configuration.properties.DatasourceProperties; +import gov.nasa.jpl.nexus.ningester.writer.*; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; +import org.springframework.data.cassandra.config.CassandraClusterFactoryBean; +import org.springframework.data.cassandra.config.CassandraSessionFactoryBean; +import org.springframework.data.cassandra.config.SchemaAction; +import org.springframework.data.cassandra.convert.CassandraConverter; +import org.springframework.data.cassandra.convert.MappingCassandraConverter; +import org.springframework.data.cassandra.core.CassandraOperations; +import org.springframework.data.cassandra.core.CassandraTemplate; +import org.springframework.data.cassandra.mapping.BasicCassandraMappingContext; +import org.springframework.data.cassandra.mapping.CassandraMappingContext; +import org.springframework.data.solr.core.SolrOperations; +import org.springframework.data.solr.core.SolrTemplate; import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; @@ -11,12 +35,139 @@ import javax.sql.DataSource; @Configuration public class DatasourceConfig { - @Bean - @Profile("embedded") - public DataSource dataSource(){ - EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder(); - return builder - .setType(EmbeddedDatabaseType.H2) - .build(); - } + private final DatasourceProperties datasourceProperties; + + @Autowired + public DatasourceConfig(DatasourceProperties datasourceProperties) { + this.datasourceProperties = datasourceProperties; + } + + @Bean + @Profile("embedded") + public DataSource dataSource() { + EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder(); + return builder + .setType(EmbeddedDatabaseType.H2) + .build(); + } + + + @Configuration + @Profile("cassandra") + class CassandraConfiguration { + @Bean + public CassandraClusterFactoryBean cluster() { + + CassandraClusterFactoryBean cluster = new CassandraClusterFactoryBean(); + cluster.setContactPoints(datasourceProperties.getCassandraStore().getContactPoints()); + cluster.setPort(datasourceProperties.getCassandraStore().getPort()); + + return cluster; + } + + @Bean + public CassandraMappingContext mappingContext() { + return new BasicCassandraMappingContext(); + } + + @Bean + public CassandraConverter converter() { + return new MappingCassandraConverter(mappingContext()); + } + + @Bean + public CassandraSessionFactoryBean session(){ + + CassandraSessionFactoryBean session = new CassandraSessionFactoryBean(); + session.setCluster(cluster().getObject()); + session.setKeyspaceName(datasourceProperties.getCassandraStore().getKeyspace()); + session.setConverter(converter()); + session.setSchemaAction(SchemaAction.NONE); + + return session; + } + + @Bean + public CassandraOperations cassandraTemplate(){ + return new CassandraTemplate(session().getObject()); + } + + @Bean + public DataStore dataStore(CassandraOperations cassandraTemplate) { + return new CassandraStore(cassandraTemplate); + } + } + + @Configuration + @Profile("dynamo") + class DynamoConfiguration { + @Bean + public AmazonDynamoDB dynamoClient() { + AmazonDynamoDB dynamoClient = new AmazonDynamoDBClient(); + dynamoClient.setRegion(Region.getRegion(Regions.fromName(datasourceProperties.getDynamoStore().getRegion()))); + return dynamoClient; + } + + @Bean + public DataStore dataStore(AmazonDynamoDB dynamoClient) { + return new DynamoStore(dynamoClient, + datasourceProperties.getDynamoStore().getTableName(), + datasourceProperties.getDynamoStore().getPrimaryKey()); + } + } + + @Configuration + @Profile("s3") + class S3Configuration { + @Bean + public AmazonS3Client s3client() { + AmazonS3Client s3Client = new AmazonS3Client(); + s3Client.setRegion(Region.getRegion(Regions.fromName(datasourceProperties.getS3Store().getRegion()))); + return s3Client; + } + + @Bean + public DataStore dataStore(AmazonS3Client s3Client) { + return new S3Store(s3Client, datasourceProperties.getS3Store().getBucketName()); + } + } + + @Configuration + @Profile("solr-standalone") + class SolrConfiguration { + @Bean + public SolrClient solrClient(){ return new HttpSolrClient(datasourceProperties.getSolrStore().getUrl() + datasourceProperties.getSolrStore().getCollection());} + + @Bean + public SolrOperations solrTemplate(SolrClient solrClient) { + return new SolrTemplate(solrClient); + } + + @Bean + public MetadataStore metadataStore(SolrOperations solrTemplate) { + return new SolrStore(solrTemplate); + } + } + + @Configuration + @Profile("solr-cloud") + class SolrCloudConfiguration { + @Bean + public SolrClient solrClient(){ + CloudSolrClient client = new CloudSolrClient(datasourceProperties.getSolrStore().getZkHost()); + client.setDefaultCollection(datasourceProperties.getSolrStore().getCollection()); + + return client; + } + + @Bean + public SolrOperations solrTemplate(SolrClient solrClient) { + return new SolrTemplate(solrClient); + } + + @Bean + public MetadataStore metadataStore(SolrOperations solrTemplate) { + return new SolrStore(solrTemplate); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDatasetName.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDatasetName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDatasetName.java deleted file mode 100644 index 8ff9230..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDatasetName.java +++ /dev/null @@ -1,24 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.configuration.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@ConfigurationProperties -@Component("addDatasetNameProperties") -public class AddDatasetName { - - private String datasetName; - - public String getDatasetName() { - return datasetName; - } - - public void setDatasetName(String datasetName) { - this.datasetName = datasetName; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDayOfYearAttribute.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDayOfYearAttribute.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDayOfYearAttribute.java deleted file mode 100644 index 7ec9513..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDayOfYearAttribute.java +++ /dev/null @@ -1,25 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.configuration.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@ConfigurationProperties -@Component("addDayOfYearAttributeProperties") -public class AddDayOfYearAttribute { - - private String regex; - - - public String getRegex() { - return regex; - } - - public void setRegex(String regex) { - this.regex = regex; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeFromGranuleName.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeFromGranuleName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeFromGranuleName.java deleted file mode 100644 index 2995bd4..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeFromGranuleName.java +++ /dev/null @@ -1,33 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.configuration.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@ConfigurationProperties -@Component("addTimeFromGranuleNameProperties") -public class AddTimeFromGranuleName { - - private String regex; - private String dateFormat; - - public String getRegex() { - return regex; - } - - public void setRegex(String regex) { - this.regex = regex; - } - - public String getDateFormat() { - return dateFormat; - } - - public void setDateFormat(String dateFormat) { - this.dateFormat = dateFormat; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java index e6ea647..9dace2e 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java @@ -5,6 +5,9 @@ package gov.nasa.jpl.nexus.ningester.configuration.properties; +import gov.nasa.jpl.nexus.ningester.datatiler.properties.SliceFileByDimension; +import gov.nasa.jpl.nexus.ningester.datatiler.properties.SliceFileByTilesDesired; +import gov.nasa.jpl.nexus.ningester.processors.properties.*; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.NestedConfigurationProperty; import org.springframework.stereotype.Component; http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java new file mode 100644 index 0000000..228ec1e --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/DatasourceProperties.java @@ -0,0 +1,47 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.configuration.properties; + +import gov.nasa.jpl.nexus.ningester.writer.properties.CassandraStore; +import gov.nasa.jpl.nexus.ningester.writer.properties.DynamoStore; +import gov.nasa.jpl.nexus.ningester.writer.properties.S3Store; +import gov.nasa.jpl.nexus.ningester.writer.properties.SolrStore; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.NestedConfigurationProperty; +import org.springframework.stereotype.Component; + +@ConfigurationProperties("datasource") +@Component +public class DatasourceProperties { + + @NestedConfigurationProperty + private final CassandraStore cassandraStore = new CassandraStore(); + + @NestedConfigurationProperty + private final DynamoStore dynamoStore = new DynamoStore(); + + @NestedConfigurationProperty + private final S3Store s3Store = new S3Store(); + + @NestedConfigurationProperty + private final SolrStore solrStore = new SolrStore(); + + public CassandraStore getCassandraStore() { + return cassandraStore; + } + + public DynamoStore getDynamoStore() { + return dynamoStore; + } + + public S3Store getS3Store() { + return s3Store; + } + + public SolrStore getSolrStore() { + return solrStore; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/GenerateTileId.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/GenerateTileId.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/GenerateTileId.java deleted file mode 100644 index 05caf6f..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/GenerateTileId.java +++ /dev/null @@ -1,24 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.configuration.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@ConfigurationProperties -@Component("generateTileIdProperties") -public class GenerateTileId { - - private String salt = ""; - - public String getSalt() { - return salt; - } - - public void setSalt(String salt) { - this.salt = salt; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProcessor.java deleted file mode 100644 index ff38025..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProcessor.java +++ /dev/null @@ -1,42 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.configuration.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -import java.net.URL; -import java.util.ArrayList; -import java.util.List; - -@ConfigurationProperties -@Component("pythonChainProcessorProperties") -public class PythonChainProcessor { - private URL baseUrl; - private String uriPath; - - private List<PythonProcessorModule> processorList = new ArrayList<>(); - - public String getUriPath() { - return uriPath; - } - - public void setUriPath(String uriPath) { - this.uriPath = uriPath; - } - - public List<PythonProcessorModule> getProcessorList() { - return processorList; - } - - public URL getBaseUrl() { - return baseUrl; - } - - public void setBaseUrl(URL baseUrl) { - this.baseUrl = baseUrl; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonProcessorModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonProcessorModule.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonProcessorModule.java deleted file mode 100644 index 287e449..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonProcessorModule.java +++ /dev/null @@ -1,27 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.configuration.properties; - -import java.util.HashMap; -import java.util.Map; - -public class PythonProcessorModule { - - private String name; - private Map<String, String> config = new HashMap<>(); - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public Map<String, String> getConfig() { - return config; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java deleted file mode 100644 index 4be4187..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java +++ /dev/null @@ -1,44 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.configuration.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -import java.util.List; - -@ConfigurationProperties -@Component("sliceFileByDimensionProperties") -public class SliceFileByDimension { - - private String sliceByDimension; - private List<String> dimensions; - private String dimensionNamePrefix = ""; - - public String getSliceByDimension() { - return sliceByDimension; - } - - public void setSliceByDimension(String sliceByDimension) { - this.sliceByDimension = sliceByDimension; - } - - public List<String> getDimensions() { - return dimensions; - } - - public void setDimensions(List<String> dimensions) { - this.dimensions = dimensions; - } - - public String getDimensionNamePrefix() { - return dimensionNamePrefix; - } - - public void setDimensionNamePrefix(String dimensionNamePrefix) { - this.dimensionNamePrefix = dimensionNamePrefix; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByTilesDesired.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByTilesDesired.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByTilesDesired.java deleted file mode 100644 index aa4d542..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByTilesDesired.java +++ /dev/null @@ -1,44 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.configuration.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.List; - -@ConfigurationProperties -@Component("sliceFileByTilesDesiredProperties") -public class SliceFileByTilesDesired { - private Integer tilesDesired; - private List<String> dimensions = new ArrayList<>(); - private String timeDimension; - - public Integer getTilesDesired() { - return tilesDesired; - } - - public void setTilesDesired(Integer tilesDesired) { - this.tilesDesired = tilesDesired; - } - - public List<String> getDimensions() { - return dimensions; - } - - public void setDimensions(List<String> dimensions) { - this.dimensions = dimensions; - } - - public String getTimeDimension() { - return timeDimension; - } - - public void setTimeDimension(String timeDimension) { - this.timeDimension = timeDimension; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByDimension.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByDimension.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByDimension.java new file mode 100644 index 0000000..0482e55 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByDimension.java @@ -0,0 +1,44 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.datatiler.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.List; + +@ConfigurationProperties +@Component("sliceFileByDimensionProperties") +public class SliceFileByDimension { + + private String sliceByDimension; + private List<String> dimensions; + private String dimensionNamePrefix = ""; + + public String getSliceByDimension() { + return sliceByDimension; + } + + public void setSliceByDimension(String sliceByDimension) { + this.sliceByDimension = sliceByDimension; + } + + public List<String> getDimensions() { + return dimensions; + } + + public void setDimensions(List<String> dimensions) { + this.dimensions = dimensions; + } + + public String getDimensionNamePrefix() { + return dimensionNamePrefix; + } + + public void setDimensionNamePrefix(String dimensionNamePrefix) { + this.dimensionNamePrefix = dimensionNamePrefix; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByTilesDesired.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByTilesDesired.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByTilesDesired.java new file mode 100644 index 0000000..cc7933b --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/properties/SliceFileByTilesDesired.java @@ -0,0 +1,44 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.datatiler.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +@ConfigurationProperties +@Component("sliceFileByTilesDesiredProperties") +public class SliceFileByTilesDesired { + private Integer tilesDesired; + private List<String> dimensions = new ArrayList<>(); + private String timeDimension; + + public Integer getTilesDesired() { + return tilesDesired; + } + + public void setTilesDesired(Integer tilesDesired) { + this.tilesDesired = tilesDesired; + } + + public List<String> getDimensions() { + return dimensions; + } + + public void setDimensions(List<String> dimensions) { + this.dimensions = dimensions; + } + + public String getTimeDimension() { + return timeDimension; + } + + public void setTimeDimension(String timeDimension) { + this.timeDimension = timeDimension; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java index 55eb051..0488562 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java @@ -9,7 +9,7 @@ package gov.nasa.jpl.nexus.ningester.processors; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import gov.nasa.jpl.nexus.ningester.configuration.properties.PythonProcessorModule; +import gov.nasa.jpl.nexus.ningester.processors.properties.PythonProcessorModule; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.core.io.Resource; import org.springframework.http.HttpEntity; http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDatasetName.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDatasetName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDatasetName.java new file mode 100644 index 0000000..7bd9af0 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDatasetName.java @@ -0,0 +1,24 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.processors.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component("addDatasetNameProperties") +public class AddDatasetName { + + private String datasetName; + + public String getDatasetName() { + return datasetName; + } + + public void setDatasetName(String datasetName) { + this.datasetName = datasetName; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDayOfYearAttribute.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDayOfYearAttribute.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDayOfYearAttribute.java new file mode 100644 index 0000000..33d8b24 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddDayOfYearAttribute.java @@ -0,0 +1,25 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.processors.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component("addDayOfYearAttributeProperties") +public class AddDayOfYearAttribute { + + private String regex; + + + public String getRegex() { + return regex; + } + + public void setRegex(String regex) { + this.regex = regex; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddTimeFromGranuleName.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddTimeFromGranuleName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddTimeFromGranuleName.java new file mode 100644 index 0000000..1042d6a --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/AddTimeFromGranuleName.java @@ -0,0 +1,33 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.processors.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component("addTimeFromGranuleNameProperties") +public class AddTimeFromGranuleName { + + private String regex; + private String dateFormat; + + public String getRegex() { + return regex; + } + + public void setRegex(String regex) { + this.regex = regex; + } + + public String getDateFormat() { + return dateFormat; + } + + public void setDateFormat(String dateFormat) { + this.dateFormat = dateFormat; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/GenerateTileId.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/GenerateTileId.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/GenerateTileId.java new file mode 100644 index 0000000..60cf75c --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/GenerateTileId.java @@ -0,0 +1,24 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.processors.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component("generateTileIdProperties") +public class GenerateTileId { + + private String salt = ""; + + public String getSalt() { + return salt; + } + + public void setSalt(String salt) { + this.salt = salt; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonChainProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonChainProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonChainProcessor.java new file mode 100644 index 0000000..ac86d99 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonChainProcessor.java @@ -0,0 +1,42 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.processors.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +@ConfigurationProperties +@Component("pythonChainProcessorProperties") +public class PythonChainProcessor { + private URL baseUrl; + private String uriPath; + + private List<PythonProcessorModule> processorList = new ArrayList<>(); + + public String getUriPath() { + return uriPath; + } + + public void setUriPath(String uriPath) { + this.uriPath = uriPath; + } + + public List<PythonProcessorModule> getProcessorList() { + return processorList; + } + + public URL getBaseUrl() { + return baseUrl; + } + + public void setBaseUrl(URL baseUrl) { + this.baseUrl = baseUrl; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonProcessorModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonProcessorModule.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonProcessorModule.java new file mode 100644 index 0000000..e48f0d5 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/properties/PythonProcessorModule.java @@ -0,0 +1,27 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.processors.properties; + +import java.util.HashMap; +import java.util.Map; + +public class PythonProcessorModule { + + private String name; + private Map<String, String> config = new HashMap<>(); + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Map<String, String> getConfig() { + return config; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStoreException.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStoreException.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStoreException.java new file mode 100644 index 0000000..9ba838e --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStoreException.java @@ -0,0 +1,28 @@ +/* + ***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ +package gov.nasa.jpl.nexus.ningester.writer; + +public class DataStoreException extends RuntimeException { + + public DataStoreException() { + } + + public DataStoreException(String message) { + super(message); + } + + public DataStoreException(String message, Throwable cause) { + super(message, cause); + } + + public DataStoreException(Throwable cause) { + super(cause); + } + + public DataStoreException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java new file mode 100644 index 0000000..4c13a14 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java @@ -0,0 +1,54 @@ +/* + ***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ +package gov.nasa.jpl.nexus.ningester.writer; + +import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; +import com.amazonaws.services.dynamodbv2.document.DynamoDB; +import com.amazonaws.services.dynamodbv2.document.Item; +import com.amazonaws.services.dynamodbv2.document.Table; +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; + +import java.util.Collection; + +/** + * Created by djsilvan on 6/26/17. + */ +public class DynamoStore implements DataStore { + + private DynamoDB dynamoDB; + private String tableName; + private String primaryKey; + + public DynamoStore(AmazonDynamoDB dynamoClient, String tableName, String primaryKey) { + dynamoDB = new DynamoDB(dynamoClient); + this.tableName = tableName; + this.primaryKey = primaryKey; + } + + public void saveData(Collection<NexusContent.NexusTile> nexusTiles) { + + Table table = dynamoDB.getTable(tableName); + + for (NexusContent.NexusTile tile : nexusTiles) { + String tileId = getTileId(tile); + byte[] tileData = getTileData(tile); + + try { + table.putItem(new Item().withPrimaryKey(primaryKey, tileId).withBinary("data", tileData)); + } catch (Exception e) { + throw new DataStoreException("Unable to add item: " + tileId, e); + } + } + } + + private String getTileId(NexusContent.NexusTile tile) { + return tile.getTile().getTileId(); + } + + private byte[] getTileData(NexusContent.NexusTile tile) { + return tile.getTile().toByteArray(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java new file mode 100644 index 0000000..a3f0e22 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java @@ -0,0 +1,62 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ +package gov.nasa.jpl.nexus.ningester.writer; + +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent.NexusTile; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.Collection; + +/** + * Created by djsilvan on 6/26/17. + */ +public class S3Store implements DataStore { + + private AmazonS3 s3; + private String bucketName; + + public S3Store(AmazonS3Client s3client, String bucketName) { + s3 = s3client; + this.bucketName = bucketName; + } + + public void saveData(Collection<NexusTile> nexusTiles) { + + for (NexusTile tile : nexusTiles) { + String tileId = getTileId(tile); + byte[] tileData = getTileData(tile); + Long contentLength = (long) tileData.length; + InputStream stream = new ByteArrayInputStream(tileData); + ObjectMetadata meta = new ObjectMetadata(); + meta.setContentLength(contentLength); + + try { + s3.putObject(new PutObjectRequest(bucketName, tileId, stream, meta)); + } catch (AmazonServiceException ase) { + throw new DataStoreException("Caught an AmazonServiceException, which means your request made it " + + "to Amazon S3, but was rejected with an error response for some reason.", ase); + } catch (AmazonClientException ace) { + throw new DataStoreException("Caught an AmazonClientException, which means the client encountered " + + "a serious internal problem while trying to communicate with S3, " + + "such as not being able to access the network.", ace); + } + } + } + + private String getTileId(NexusTile tile) { + return tile.getTile().getTileId(); + } + + private byte[] getTileData(NexusTile tile) { + return tile.getTile().toByteArray(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java index 71f4ef0..1706b77 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java @@ -20,7 +20,7 @@ import java.text.SimpleDateFormat; import java.util.*; import java.util.stream.Collectors; -public class SolrStore { +public class SolrStore implements MetadataStore { private Environment environment; private SolrOperations solr; @@ -44,6 +44,7 @@ public class SolrStore { this.environment = environment; } + @Override public void saveMetadata(Collection<NexusContent.NexusTile> nexusTiles) { List<SolrInputDocument> solrdocs = nexusTiles.stream() http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java new file mode 100644 index 0000000..c6da68e --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java @@ -0,0 +1,42 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.writer.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties("cassandra") +@Component("cassandraStoreProperties") +public class CassandraStore { + + private String contactPoints; + private String keyspace; + private Integer port; + + public String getContactPoints() { + return contactPoints; + } + + public void setContactPoints(String contactPoints) { + this.contactPoints = contactPoints; + } + + public String getKeyspace() { + return keyspace; + } + + public void setKeyspace(String keyspace) { + this.keyspace = keyspace; + } + + public Integer getPort() { + return port; + } + + public void setPort(Integer port) { + this.port = port; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/DynamoStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/DynamoStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/DynamoStore.java new file mode 100644 index 0000000..455396e --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/DynamoStore.java @@ -0,0 +1,44 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.writer.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties("dynamo") +@Component("dynamoStoreProperties") +public class DynamoStore { + + private String tableName; + + private String region; + + private String primaryKey = "tile_id"; + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getPrimaryKey() { + return primaryKey; + } + + public void setPrimaryKey(String primaryKey) { + this.primaryKey = primaryKey; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/S3Store.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/S3Store.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/S3Store.java new file mode 100644 index 0000000..367e844 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/S3Store.java @@ -0,0 +1,34 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.writer.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties("s3") +@Component("s3StoreProperties") +public class S3Store { + + private String bucketName; + + private String region; + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getBucketName() { + return bucketName; + } + + public void setBucketName(String bucketName) { + this.bucketName = bucketName; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java new file mode 100644 index 0000000..3dbb41d --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java @@ -0,0 +1,44 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.writer.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties("solr") +@Component("solrStoreProperties") +public class SolrStore { + + + private String url; + private String zkHost; + private String collection; + + + public String getCollection() { + return collection; + } + + public void setCollection(String collection) { + this.collection = collection; + } + + public String getZkHost() { + return zkHost; + } + + public void setZkHost(String zkHost) { + this.zkHost = zkHost; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/resources/application.properties ---------------------------------------------------------------------- diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties deleted file mode 100644 index e69de29..0000000 http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/a52c618e/src/main/resources/application.yml ---------------------------------------------------------------------- diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 0000000..b536524 --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,5 @@ +spring: + autoconfigure: + exclude: + - org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration + - org.springframework.boot.autoconfigure.data.cassandra.CassandraDataAutoConfiguration \ No newline at end of file
