http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java deleted file mode 100644 index 6869f08..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - **************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.writer; - -import org.apache.sdap.nexusproto.NexusTile; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; - -public class NexusWriter { - - private static final Logger log = LoggerFactory.getLogger(NexusWriter.class); - private MetadataStore metadataStore; - private DataStore dataStore; - - public NexusWriter(MetadataStore metadataStore, DataStore dataStore) { - this.metadataStore = metadataStore; - this.dataStore = dataStore; - } - - public void saveToNexus(List<? extends NexusTile> nexusTiles) { - if (nexusTiles.size() > 0) { - metadataStore.saveMetadata(nexusTiles); - - try { - dataStore.saveData(nexusTiles); - - } catch (RuntimeException e) { - try { - metadataStore.deleteMetadata(nexusTiles); - } catch (RuntimeException e2) { - log.error("During exception while saving data, could not rollback metadata", e2); - } - throw e; - } - } - } - -}
http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java deleted file mode 100644 index 6dc75f4..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java +++ /dev/null @@ -1,62 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ -package gov.nasa.jpl.nexus.ningester.writer; - -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectRequest; -import org.apache.sdap.nexusproto.NexusTile; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.util.List; - -/** - * Created by djsilvan on 6/26/17. - */ -public class S3Store implements DataStore { - - private AmazonS3 s3; - private String bucketName; - - public S3Store(AmazonS3Client s3client, String bucketName) { - s3 = s3client; - this.bucketName = bucketName; - } - - public void saveData(List<? extends NexusTile> nexusTiles) { - - for (NexusTile tile : nexusTiles) { - String tileId = getTileId(tile); - byte[] tileData = getTileData(tile); - Long contentLength = (long) tileData.length; - InputStream stream = new ByteArrayInputStream(tileData); - ObjectMetadata meta = new ObjectMetadata(); - meta.setContentLength(contentLength); - - try { - s3.putObject(new PutObjectRequest(bucketName, tileId, stream, meta)); - } catch (AmazonServiceException ase) { - throw new DataStoreException("Caught an AmazonServiceException, which means your request made it " - + "to Amazon S3, but was rejected with an error response for some reason.", ase); - } catch (AmazonClientException ace) { - throw new DataStoreException("Caught an AmazonClientException, which means the client encountered " - + "a serious internal problem while trying to communicate with S3, " - + "such as not being able to access the network.", ace); - } - } - } - - private String getTileId(NexusTile tile) { - return tile.getTile().getTileId(); - } - - private byte[] getTileData(NexusTile tile) { - return tile.getTile().toByteArray(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java deleted file mode 100644 index 0a48733..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java +++ /dev/null @@ -1,166 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.writer; - -import org.apache.sdap.nexusproto.NexusTile; -import org.apache.sdap.nexusproto.TileSummary; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.SolrInputField; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.solr.core.SolrOperations; - -import java.math.BigDecimal; -import java.nio.file.Paths; -import java.text.SimpleDateFormat; -import java.util.*; -import java.util.stream.Collectors; - -public class SolrStore implements MetadataStore { - - //TODO This will be refactored at some point to be dynamic per-message. Or maybe per-group. - private static final String TABLE_NAME = "sea_surface_temp"; - private static final SimpleDateFormat iso = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); - - static { - iso.setTimeZone(TimeZone.getTimeZone("UTC")); - } - - private Integer commitWithin = 1000; - private Integer geoPrecision = 3; - private String collection = "nexustiles"; - private SolrOperations solr; - private Logger log = LoggerFactory.getLogger(SolrStore.class); - - public SolrStore(SolrOperations solr) { - this.solr = solr; - } - - @Override - public void saveMetadata(List<? extends NexusTile> nexusTiles) { - - List<SolrInputDocument> solrdocs = nexusTiles.stream() - .map(nexusTile -> getSolrDocFromTileSummary(nexusTile.getSummary())) - .collect(Collectors.toList()); - solr.saveDocuments(this.collection, solrdocs, commitWithin); - } - - @Override - public void deleteMetadata(List<? extends NexusTile> nexusTiles) { - - List<String> tileIds = nexusTiles.stream() - .map(nexusTile -> nexusTile.getSummary().getDatasetName() + "!" + nexusTile.getSummary().getTileId()) - .collect(Collectors.toList()); - solr.deleteById(this.collection, tileIds); - } - - public SolrInputDocument getSolrDocFromTileSummary(TileSummary summary) { - - TileSummary.BBox bbox = summary.getBbox(); - TileSummary.DataStats stats = summary.getStats(); - - Calendar startCal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - startCal.setTime(new Date(stats.getMinTime() * 1000)); - Calendar endCal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - endCal.setTime(new Date(stats.getMaxTime() * 1000)); - - String minTime = iso.format(startCal.getTime()); - String maxTime = iso.format(endCal.getTime()); - - String geo = determineGeo(summary); - - String granuleFileName = Paths.get(summary.getGranule()).getFileName().toString(); - - SolrInputDocument inputDocument = new SolrInputDocument(); - inputDocument.addField("table_s", TABLE_NAME); - inputDocument.addField("geo", geo); - inputDocument.addField("id", summary.getTileId()); - inputDocument.addField("solr_id_s", summary.getDatasetName() + "!" + summary.getTileId()); - inputDocument.addField("dataset_id_s", summary.getDatasetUuid()); - inputDocument.addField("sectionSpec_s", summary.getSectionSpec()); - inputDocument.addField("dataset_s", summary.getDatasetName()); - inputDocument.addField("granule_s", granuleFileName); - inputDocument.addField("tile_var_name_s", summary.getDataVarName()); - inputDocument.addField("tile_min_lon", bbox.getLonMin()); - inputDocument.addField("tile_max_lon", bbox.getLonMax()); - inputDocument.addField("tile_min_lat", bbox.getLatMin()); - inputDocument.addField("tile_max_lat", bbox.getLatMax()); - inputDocument.addField("tile_min_time_dt", minTime); - inputDocument.addField("tile_max_time_dt", maxTime); - inputDocument.addField("tile_min_val_d", stats.getMin()); - inputDocument.addField("tile_max_val_d", stats.getMax()); - inputDocument.addField("tile_avg_val_d", stats.getMean()); - inputDocument.addField("tile_count_i", Long.valueOf(stats.getCount()).intValue()); - - summary.getGlobalAttributesList().forEach(attribute -> - inputDocument.addField(attribute.getName(), attribute.getValuesCount() == 1 ? attribute.getValues(0) : attribute.getValuesList()) - ); - return inputDocument; - } - - private String determineGeo(TileSummary summary) { - //Solr cannot index a POLYGON where all corners are the same point or when there are only 2 distinct points (line). - //Solr is configured for a specific precision so we need to round to that precision before checking equality. - Integer geoPrecision = this.geoPrecision; - - BigDecimal latMin = BigDecimal.valueOf(summary.getBbox().getLatMin()).setScale(geoPrecision, BigDecimal.ROUND_HALF_UP); - BigDecimal latMax = BigDecimal.valueOf(summary.getBbox().getLatMax()).setScale(geoPrecision, BigDecimal.ROUND_HALF_UP); - BigDecimal lonMin = BigDecimal.valueOf(summary.getBbox().getLonMin()).setScale(geoPrecision, BigDecimal.ROUND_HALF_UP); - BigDecimal lonMax = BigDecimal.valueOf(summary.getBbox().getLonMax()).setScale(geoPrecision, BigDecimal.ROUND_HALF_UP); - - String geo; - //If lat min = lat max and lon min = lon max, index the 'geo' bounding box as a POINT instead of a POLYGON - if (latMin.equals(latMax) && lonMin.equals(lonMax)) { - geo = "POINT(" + lonMin + " " + latMin + ")"; - log.debug("{}\t{}[{}] geo={}", summary.getTileId(), summary.getGranule(), summary.getSectionSpec(), geo); - } - //If lat min = lat max but lon min != lon max, then we essentially have a line. - else if (latMin.equals(latMax)) { - geo = "LINESTRING (" + lonMin + " " + latMin + ", " + lonMax + " " + latMin + ")"; - log.debug("{}\t{}[{}] geo={}", summary.getTileId(), summary.getGranule(), summary.getSectionSpec(), geo); - } - //Same if lon min = lon max but lat min != lat max - else if (lonMin.equals(lonMax)) { - geo = "LINESTRING (" + lonMin + " " + latMin + ", " + lonMin + " " + latMax + ")"; - log.debug("{}\t{}[{}] geo={}", summary.getTileId(), summary.getGranule(), summary.getSectionSpec(), geo); - } - //All other cases should use POLYGON - else { - geo = "POLYGON((" + - lonMin + " " + latMin + ", " + - lonMax + " " + latMin + ", " + - lonMax + " " + latMax + ", " + - lonMin + " " + latMax + ", " + - lonMin + " " + latMin + "))"; - } - - return geo; - } - - private SolrInputDocument toSolrInputDocument(Map<String, Object> doc) { - SolrInputDocument solrDoc = new SolrInputDocument(); - - solrDoc.putAll(doc.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> { - SolrInputField field = new SolrInputField(entry.getKey()); - field.setValue(entry.getValue(), 0); - return field; - }))); - - return solrDoc; - } - - public void setCollection(String collection) { - this.collection = collection; - } - - public void setCommitWithin(Integer commitWithin) { - this.commitWithin = commitWithin; - } - - public void setGeoPrecision(Integer geoPrecision) { - this.geoPrecision = geoPrecision; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java deleted file mode 100644 index a2d40d5..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/CassandraStore.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - **************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.writer.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@ConfigurationProperties -@Component("cassandraStoreProperties") -public class CassandraStore { - - private String tableName = "sea_surface_temp"; - - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/DynamoStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/DynamoStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/DynamoStore.java deleted file mode 100644 index 455396e..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/DynamoStore.java +++ /dev/null @@ -1,44 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.writer.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@ConfigurationProperties("dynamo") -@Component("dynamoStoreProperties") -public class DynamoStore { - - private String tableName; - - private String region; - - private String primaryKey = "tile_id"; - - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public String getRegion() { - return region; - } - - public void setRegion(String region) { - this.region = region; - } - - public String getPrimaryKey() { - return primaryKey; - } - - public void setPrimaryKey(String primaryKey) { - this.primaryKey = primaryKey; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/S3Store.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/S3Store.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/S3Store.java deleted file mode 100644 index 367e844..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/S3Store.java +++ /dev/null @@ -1,34 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.writer.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@ConfigurationProperties("s3") -@Component("s3StoreProperties") -public class S3Store { - - private String bucketName; - - private String region; - - public String getRegion() { - return region; - } - - public void setRegion(String region) { - this.region = region; - } - - public String getBucketName() { - return bucketName; - } - - public void setBucketName(String bucketName) { - this.bucketName = bucketName; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java deleted file mode 100644 index ff69008..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/properties/SolrStore.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - ***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.writer.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@ConfigurationProperties -@Component("solrStoreProperties") -public class SolrStore { - - - private Integer commitWithin = 1000; - private Integer geoPrecision = 3; - private String collection = "nexustiles"; - - - public String getCollection() { - return collection; - } - - public void setCollection(String collection) { - this.collection = collection; - } - - public Integer getCommitWithin() { - return commitWithin; - } - - public void setCommitWithin(Integer commitWithin) { - this.commitWithin = commitWithin; - } - - public Integer getGeoPrecision() { - return geoPrecision; - } - - public void setGeoPrecision(Integer geoPrecision) { - this.geoPrecision = geoPrecision; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/NingesterApplication.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/NingesterApplication.java b/src/main/java/org/apache/sdap/ningester/NingesterApplication.java new file mode 100644 index 0000000..88071b1 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/NingesterApplication.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sdap.ningester; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.ApplicationContext; + +@SpringBootApplication +public class NingesterApplication { + + public static void main(String[] args) { + + ApplicationContext context = SpringApplication.run(NingesterApplication.class, args); + SpringApplication.exit(context); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/configuration/AppConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/configuration/AppConfig.java b/src/main/java/org/apache/sdap/ningester/configuration/AppConfig.java new file mode 100644 index 0000000..5a0c98e --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/configuration/AppConfig.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.configuration; + +import org.apache.sdap.nexusproto.NexusTile; +import org.apache.sdap.ningester.configuration.properties.ApplicationProperties; +import org.apache.sdap.ningester.datatiler.FileSlicer; +import org.apache.sdap.ningester.datatiler.SliceFileByDimension; +import org.apache.sdap.ningester.datatiler.SliceFileByTilesDesired; +import org.apache.sdap.ningester.http.NexusTileConverter; +import org.apache.sdap.ningester.processors.*; +import org.apache.sdap.ningester.writer.DataStore; +import org.apache.sdap.ningester.writer.MetadataStore; +import org.apache.sdap.ningester.writer.NexusWriter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.MediaType; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.DefaultUriTemplateHandler; + +import java.util.Collections; +import java.util.List; + +@Configuration +@EnableConfigurationProperties({ApplicationProperties.class}) +public class AppConfig { + + private final ApplicationProperties applicationProperties; + + @Autowired + public AppConfig(ApplicationProperties applicationProperties) { + this.applicationProperties = applicationProperties; + } + + @Bean + @ConditionalOnProperty(prefix = "ningester", name = "tile_slicer", havingValue = "sliceFileByTilesDesired") + @Qualifier("fileSlicer") + protected FileSlicer sliceFileByTilesDesired() { + SliceFileByTilesDesired fileSlicer = new SliceFileByTilesDesired(); + fileSlicer.setDimensions(applicationProperties.getSliceFileByTilesDesired().getDimensions()); + fileSlicer.setTilesDesired(applicationProperties.getSliceFileByTilesDesired().getTilesDesired()); + fileSlicer.setTimeDimension(applicationProperties.getSliceFileByTilesDesired().getTimeDimension()); + return fileSlicer; + } + + @Bean + @ConditionalOnProperty(prefix = "ningester", name = "tile_slicer", havingValue = "sliceFileByDimension") + @Qualifier("fileSlicer") + protected FileSlicer sliceFileByDimension() { + SliceFileByDimension fileSlicer = new SliceFileByDimension(); + fileSlicer.setDimensions(applicationProperties.getSliceFileByDimension().getDimensions()); + fileSlicer.setSliceByDimension(applicationProperties.getSliceFileByDimension().getSliceByDimension()); + fileSlicer.setDimensionNamePrefix(applicationProperties.getSliceFileByDimension().getDimensionNamePrefix()); + return fileSlicer; + } + + @Bean + protected HttpMessageConverter nexusTileConverter() { + NexusTileConverter converter = new NexusTileConverter(); + converter.setSupportedMediaTypes(Collections.singletonList(MediaType.APPLICATION_OCTET_STREAM)); + return converter; + } + + @Bean + @ConditionalOnProperty(prefix = "ningester.pythonChainProcessor", name = "enabled") + protected RestTemplate restTemplate(HttpMessageConverter nexusTileConverter) { + RestTemplate template = new RestTemplate(); + + DefaultUriTemplateHandler uriTemplateHandler = new DefaultUriTemplateHandler(); + uriTemplateHandler.setBaseUrl(applicationProperties.getPythonChainProcessor().getBaseUrl().toString()); + template.setUriTemplateHandler(uriTemplateHandler); + + List<HttpMessageConverter<?>> converters = template.getMessageConverters(); + converters.add(nexusTileConverter); + template.setMessageConverters(converters); + + return template; + } + + @Bean + public MetadataStore metadataStore() { + return new MetadataStore() { + @Override + public void saveMetadata(List<? extends NexusTile> nexusTiles) { + } + + @Override + public void deleteMetadata(List<? extends NexusTile> nexusTiles) { + } + }; + } + + @Bean + public DataStore dataStore() { + return nexusTiles -> { + }; + } + + @Bean + public NexusWriter nexusWriter(MetadataStore metadataStore, DataStore dataStore) { + return new NexusWriter(metadataStore, dataStore); + } + + /* + * Item Processor beans defined below + */ + @Bean + @ConditionalOnProperty(prefix = "ningester.addDatasetName", name = "enabled") + protected AddDatasetName addDatasetNameBean() { + + AddDatasetName processor = new AddDatasetName(applicationProperties.getAddDatasetName().getDatasetName()); + return processor; + } + + @Bean + @ConditionalOnProperty(prefix = "ningester.addDayOfYearAttribute", name = "enabled") + protected AddDayOfYearAttribute addDayOfYearAttributeBean() { + + AddDayOfYearAttribute processor = new AddDayOfYearAttribute(applicationProperties.getAddDayOfYearAttribute().getRegex()); + return processor; + } + + @Bean + @ConditionalOnProperty(prefix = "ningester.addTimeFromGranuleName", name = "enabled") + protected AddTimeFromGranuleName addTimeFromGranuleNameBean() { + + AddTimeFromGranuleName processor = new AddTimeFromGranuleName(applicationProperties.getAddTimeFromGranuleName().getRegex(), applicationProperties.getAddTimeFromGranuleName().getDateFormat()); + return processor; + } + + @Bean + @ConditionalOnProperty(prefix = "ningester.generateTileId", name = "enabled") + protected GenerateTileId generateTileIdBean() { + + GenerateTileId processor = new GenerateTileId(); + processor.setSalt(applicationProperties.getGenerateTileId().getSalt()); + return processor; + } + + @Bean + @ConditionalOnProperty(prefix = "ningester.pythonChainProcessor", name = "enabled") + protected PythonChainProcessor pythonChainProcessorBean(RestTemplate restTemplate) { + PythonChainProcessor processor = new PythonChainProcessor(restTemplate); + processor.setProcessorList(applicationProperties.getPythonChainProcessor().getProcessorList()); + processor.setUriPath(applicationProperties.getPythonChainProcessor().getUriPath()); + + return processor; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/configuration/BatchConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/configuration/BatchConfig.java b/src/main/java/org/apache/sdap/ningester/configuration/BatchConfig.java new file mode 100644 index 0000000..34ffb9c --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/configuration/BatchConfig.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sdap.ningester.configuration; + +import org.apache.sdap.nexusproto.NexusTile; +import org.apache.sdap.ningester.configuration.properties.ApplicationProperties; +import org.apache.sdap.ningester.datatiler.FileSlicer; +import org.apache.sdap.ningester.datatiler.NetCDFItemReader; +import org.apache.sdap.ningester.processors.*; +import org.apache.sdap.ningester.writer.NexusWriter; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.JobScope; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemStreamReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.core.io.Resource; +import org.springframework.core.io.ResourceLoader; + +@Configuration +@EnableBatchProcessing +@Import(AppConfig.class) +public class BatchConfig { + + public static final String NINGESTER_JOB_NAME = "ningester"; + + @Autowired + protected JobBuilderFactory jobs; + + @Autowired + protected StepBuilderFactory steps; + + @Autowired + protected ApplicationProperties applicationProperties; + + @Autowired + protected ApplicationContext context; + + @Bean + public Job job(@Qualifier("ingestGranule") Step step1) { + return jobs.get(NINGESTER_JOB_NAME).start(step1).build(); + } + + @Bean + @JobScope + protected Resource granule(ResourceLoader resourceLoader, @Value("#{jobParameters['granule']}") String granuleLocation) { + return resourceLoader.getResource(granuleLocation); + } + + @Bean + @JobScope + protected ItemStreamReader<NexusTile> reader(FileSlicer fileSlicer, Resource granule) { + NetCDFItemReader reader = new NetCDFItemReader(fileSlicer); + reader.setResource(granule); + return reader; + } + + @Bean + @JobScope + protected ItemProcessor<NexusTile, NexusTile> processor() { + return new CompositeItemProcessor<>(applicationProperties.getTileProcessors()); + } + + @Bean + @JobScope + protected ItemWriter<NexusTile> writer(NexusWriter nexusWriter) { + return nexusWriter::saveToNexus; + } + + @Bean + @JobScope + protected Step ingestGranule(ItemStreamReader<NexusTile> reader, ItemProcessor<NexusTile, NexusTile> processor, ItemWriter<NexusTile> writer) { + return steps.get("ingestGranule") + .<NexusTile, NexusTile>chunk(10) + .reader(reader) + .processor(processor) + .writer(writer).build(); + } + + /* + * Item Processor beans defined below + */ + @Bean + @ConditionalOnBean(AddDatasetName.class) + protected ItemProcessor<NexusTile, NexusTile> addDatasetName(AddDatasetName addDatasetNameBean) { + return addDatasetNameBean::addDatasetName; + } + + @Bean + @ConditionalOnBean(AddDayOfYearAttribute.class) + protected ItemProcessor<NexusTile, NexusTile> addDayOfYearAttribute(AddDayOfYearAttribute addDayOfYearAttributeBean) { + return addDayOfYearAttributeBean::setDayOfYearFromGranuleName; + } + + @Bean + @ConditionalOnBean(AddTimeFromGranuleName.class) + protected ItemProcessor<NexusTile, NexusTile> addTimeFromGranuleName(AddTimeFromGranuleName addTimeFromGranuleNameBean) { + return addTimeFromGranuleNameBean::setTimeFromGranuleName; + } + + @Bean + @ConditionalOnBean(GenerateTileId.class) + protected ItemProcessor<NexusTile, NexusTile> generateTileId(GenerateTileId generateTileIdBean) { + return generateTileIdBean::addTileId; + } + + @Bean + @ConditionalOnBean(PythonChainProcessor.class) + protected ItemProcessor<NexusTile, NexusTile> pythonChainProcessor(PythonChainProcessor pythonChainProcessorBean) { + return pythonChainProcessorBean::nexusTileProcessor; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/configuration/DatasourceConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/configuration/DatasourceConfig.java b/src/main/java/org/apache/sdap/ningester/configuration/DatasourceConfig.java new file mode 100644 index 0000000..c8818c5 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/configuration/DatasourceConfig.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sdap.ningester.configuration; + +import com.amazonaws.regions.Region; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; +import com.amazonaws.services.s3.AmazonS3Client; +import org.apache.sdap.ningester.configuration.properties.DatasourceProperties; +import org.apache.sdap.ningester.writer.*; +import org.apache.solr.client.solrj.SolrClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration; +import org.springframework.boot.autoconfigure.data.cassandra.CassandraDataAutoConfiguration; +import org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Profile; +import org.springframework.data.cassandra.core.CassandraTemplate; +import org.springframework.data.solr.core.SolrOperations; +import org.springframework.data.solr.core.SolrTemplate; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class DatasourceConfig { + + @Bean + @Profile("embedded") + public DataSource dataSource() { + EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder(); + return builder + .setType(EmbeddedDatabaseType.H2) + .build(); + } + + + @Configuration + @Profile("cassandra") + @Import({CassandraDataAutoConfiguration.class, CassandraAutoConfiguration.class}) + static class CassandraConfiguration { + + @Bean + public DataStore dataStore(CassandraTemplate cassandraTemplate) { + return new CassandraStore(cassandraTemplate); + } + } + + @Configuration + @Profile("dynamo") + static class DynamoConfiguration { + + @Autowired + private DatasourceProperties datasourceProperties; + + @Bean + public AmazonDynamoDB dynamoClient() { + AmazonDynamoDB dynamoClient = new AmazonDynamoDBClient(); + dynamoClient.setRegion(Region.getRegion(Regions.fromName(datasourceProperties.getDynamoStore().getRegion()))); + return dynamoClient; + } + + @Bean + public DataStore dataStore(AmazonDynamoDB dynamoClient) { + return new DynamoStore(dynamoClient, + datasourceProperties.getDynamoStore().getTableName(), + datasourceProperties.getDynamoStore().getPrimaryKey()); + } + } + + @Configuration + @Profile("s3") + static class S3Configuration { + @Autowired + private DatasourceProperties datasourceProperties; + + @Bean + public AmazonS3Client s3client() { + AmazonS3Client s3Client = new AmazonS3Client(); + s3Client.setRegion(Region.getRegion(Regions.fromName(datasourceProperties.getS3Store().getRegion()))); + return s3Client; + } + + @Bean + public DataStore dataStore(AmazonS3Client s3Client) { + return new S3Store(s3Client, datasourceProperties.getS3Store().getBucketName()); + } + } + + @Configuration + @Profile("solr") + @Import({SolrAutoConfiguration.class}) + static class SolrConfiguration { + + @Autowired + private DatasourceProperties datasourceProperties; + + @Bean + public SolrOperations solrTemplate(SolrClient solrClient) { + return new SolrTemplate(solrClient); + } + + + @Bean + public MetadataStore metadataStore(SolrOperations solrTemplate) { + SolrStore solrStore = new SolrStore(solrTemplate); + solrStore.setCollection(datasourceProperties.getSolrStore().getCollection()); + solrStore.setCommitWithin(datasourceProperties.getSolrStore().getCommitWithin()); + solrStore.setGeoPrecision(datasourceProperties.getSolrStore().getGeoPrecision()); + + return solrStore; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/configuration/properties/ApplicationProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/configuration/properties/ApplicationProperties.java b/src/main/java/org/apache/sdap/ningester/configuration/properties/ApplicationProperties.java new file mode 100644 index 0000000..a970f80 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/configuration/properties/ApplicationProperties.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.configuration.properties; + +import org.apache.sdap.ningester.datatiler.properties.SliceFileByDimension; +import org.apache.sdap.ningester.datatiler.properties.SliceFileByTilesDesired; +import org.apache.sdap.ningester.processors.properties.*; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.NestedConfigurationProperty; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +@ConfigurationProperties("ningester") +@Component +public class ApplicationProperties { + + @NestedConfigurationProperty + private final SliceFileByDimension sliceFileByDimension = new SliceFileByDimension(); + @NestedConfigurationProperty + private final SliceFileByTilesDesired sliceFileByTilesDesired = new SliceFileByTilesDesired(); + @NestedConfigurationProperty + private final AddDayOfYearAttribute addDayOfYearAttribute = new AddDayOfYearAttribute(); + @NestedConfigurationProperty + private final AddTimeFromGranuleName addTimeFromGranuleName = new AddTimeFromGranuleName(); + @NestedConfigurationProperty + private final GenerateTileId generateTileId = new GenerateTileId(); + @NestedConfigurationProperty + private final PythonChainProcessor pythonChainProcessor = new PythonChainProcessor(); + @NestedConfigurationProperty + private final AddDatasetName addDatasetName = new AddDatasetName(); + private String tileSlicer; + private List<String> tileProcessors = new ArrayList<>(); + + public PythonChainProcessor getPythonChainProcessor() { + return pythonChainProcessor; + } + + public List<String> getTileProcessors() { + return tileProcessors; + } + + public AddDayOfYearAttribute getAddDayOfYearAttribute() { + return addDayOfYearAttribute; + } + + public AddTimeFromGranuleName getAddTimeFromGranuleName() { + return addTimeFromGranuleName; + } + + public GenerateTileId getGenerateTileId() { + return generateTileId; + } + + public AddDatasetName getAddDatasetName() { + return addDatasetName; + } + + public String getTileSlicer() { + return tileSlicer; + } + + public void setTileSlicer(String tileSlicer) { + this.tileSlicer = tileSlicer; + } + + public SliceFileByTilesDesired getSliceFileByTilesDesired() { + return sliceFileByTilesDesired; + } + + public SliceFileByDimension getSliceFileByDimension() { + return sliceFileByDimension; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/configuration/properties/DatasourceProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/configuration/properties/DatasourceProperties.java b/src/main/java/org/apache/sdap/ningester/configuration/properties/DatasourceProperties.java new file mode 100644 index 0000000..b5e5a96 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/configuration/properties/DatasourceProperties.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.configuration.properties; + +import org.apache.sdap.ningester.writer.properties.CassandraStore; +import org.apache.sdap.ningester.writer.properties.DynamoStore; +import org.apache.sdap.ningester.writer.properties.S3Store; +import org.apache.sdap.ningester.writer.properties.SolrStore; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.NestedConfigurationProperty; +import org.springframework.stereotype.Component; + +@ConfigurationProperties("datasource") +@Component +public class DatasourceProperties { + + @NestedConfigurationProperty + private final CassandraStore cassandraStore = new CassandraStore(); + + @NestedConfigurationProperty + private final DynamoStore dynamoStore = new DynamoStore(); + + @NestedConfigurationProperty + private final S3Store s3Store = new S3Store(); + + @NestedConfigurationProperty + private final SolrStore solrStore = new SolrStore(); + + public DynamoStore getDynamoStore() { + return dynamoStore; + } + + public S3Store getS3Store() { + return s3Store; + } + + public SolrStore getSolrStore() { + return solrStore; + } + + public CassandraStore getCassandraStore() { + return cassandraStore; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/datatiler/FileSlicer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/datatiler/FileSlicer.java b/src/main/java/org/apache/sdap/ningester/datatiler/FileSlicer.java new file mode 100644 index 0000000..4d86b5e --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/datatiler/FileSlicer.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.datatiler; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +public interface FileSlicer { + + List<String> generateSlices(File inputfile) throws IOException; + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/datatiler/NetCDFItemReader.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/datatiler/NetCDFItemReader.java b/src/main/java/org/apache/sdap/ningester/datatiler/NetCDFItemReader.java new file mode 100644 index 0000000..abad00a --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/datatiler/NetCDFItemReader.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sdap.ningester.datatiler; + +import org.apache.sdap.nexusproto.NexusTile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.item.ItemStreamException; +import org.springframework.batch.item.UnexpectedInputException; +import org.springframework.batch.item.file.ResourceAwareItemReaderItemStream; +import org.springframework.core.io.Resource; +import ucar.nc2.dataset.NetcdfDataset; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class NetCDFItemReader implements ResourceAwareItemReaderItemStream<NexusTile> { + + static final String CURRENT_TILE_SPEC_INDEX_KEY = "current.tile.spec.index"; + private static final Logger log = LoggerFactory.getLogger(NetCDFItemReader.class); + private List<String> tileSpecList; + private Integer currentTileSpecIndex; + + private Resource netCDFResource; + private NetcdfDataset ds; + private FileSlicer fileSlicer; + + /** + * Constructor + * + * @param fileSlicer Object responsible for slicing the NetCDF file into tiles. + */ + public NetCDFItemReader(FileSlicer fileSlicer) { + this.fileSlicer = fileSlicer; + } + + @Override + public NexusTile read() { + if (this.currentTileSpecIndex == this.tileSpecList.size()) { + //End of stream + return null; + } + String currentSpec = this.tileSpecList.get(this.currentTileSpecIndex); + + URL netCDFUrl = null; + try { + netCDFUrl = this.netCDFResource.getURL(); + } catch (IOException e) { + throw new UnexpectedInputException("Generic IOException", e); + } + + NexusTile.Builder nexusTileBuilder = NexusTile.newBuilder(); + nexusTileBuilder.getSummaryBuilder() + .setSectionSpec(currentSpec) + .setGranule(netCDFUrl.toString()); + + this.currentTileSpecIndex++; + return nexusTileBuilder.build(); + } + + @Override + public void open(ExecutionContext executionContext) throws ItemStreamException { + + File netCDFFile = null; + try { + netCDFFile = this.netCDFResource.getFile(); + } catch (IOException e) { + throw new ItemStreamException(e); + } + + //Every time we open the file we generate the tile specs according to the given file slicer + try { + this.tileSpecList = fileSlicer.generateSlices(netCDFFile); + } catch (IOException e) { + throw new ItemStreamException(e); + } + log.debug("Generated tile specifications for {}\nINDEX\tTILE SPECIFICATION\n{}", netCDFFile.getName(), + IntStream.range(0, this.tileSpecList.size()) + .mapToObj(i -> i + "\t" + this.tileSpecList.get(i)) + .collect(Collectors.joining("\n"))); + + if (!executionContext.containsKey(CURRENT_TILE_SPEC_INDEX_KEY)) { + //Start at index 0 + this.currentTileSpecIndex = 0; + executionContext.putInt(CURRENT_TILE_SPEC_INDEX_KEY, this.currentTileSpecIndex); + } else { + //Start at index location from context + this.currentTileSpecIndex = executionContext.getInt(CURRENT_TILE_SPEC_INDEX_KEY); + } + + //Open the resource + try { + this.ds = NetcdfDataset.openDataset(netCDFFile.getAbsolutePath()); + } catch (IOException e) { + throw new ItemStreamException(e); + } + + } + + @Override + public void update(ExecutionContext executionContext) throws ItemStreamException { + + executionContext.putInt(CURRENT_TILE_SPEC_INDEX_KEY, this.currentTileSpecIndex); + } + + @Override + public void close() throws ItemStreamException { + + try { + this.ds.close(); + } catch (IOException e) { + throw new ItemStreamException(e); + } + + } + + @Override + public void setResource(Resource resource) { + this.netCDFResource = resource; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/datatiler/SliceFileByDimension.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/datatiler/SliceFileByDimension.java b/src/main/java/org/apache/sdap/ningester/datatiler/SliceFileByDimension.java new file mode 100644 index 0000000..d20f75e --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/datatiler/SliceFileByDimension.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sdap.ningester.datatiler; + +import com.google.common.base.Strings; +import com.google.common.collect.Sets; +import ucar.nc2.Dimension; +import ucar.nc2.Variable; +import ucar.nc2.dataset.NetcdfDataset; + +import java.io.File; +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class SliceFileByDimension implements FileSlicer { + + private String sliceByDimension; + private List<String> dimensions; + private String dimensionNamePrefix; + + public void setDimensions(List<String> dims) { + this.dimensions = dims; + } + + public void setSliceByDimension(String sliceBy) { + this.sliceByDimension = sliceBy; + } + + public void setDimensionNamePrefix(String dimensionNamePrefix) { + this.dimensionNamePrefix = dimensionNamePrefix; + } + + public List<String> generateSlices(File inputfile) throws IOException { + + boolean isInteger = false; + try { + Integer.parseInt(this.sliceByDimension); + isInteger = true; + } catch (NumberFormatException e) { + //ignore + } + return (isInteger) ? indexedDimensionSlicing(inputfile) : namedDimensionSlicing(inputfile); + } + + List<String> indexedDimensionSlicing(File inputfile) throws IOException { + + // This is sort of a hack to help the python netcdf library. When you try to get dimensions by name and they are unnamed, the + // python library uses 'phony_dim_' then the index of the dimension as the dimension name. Weird, I know. + if (Strings.isNullOrEmpty(this.dimensionNamePrefix)) { + this.dimensionNamePrefix = "phony_dim_"; + } + Map<String, Integer> dimensionNameToLength; + try (NetcdfDataset ds = NetcdfDataset.openDataset(inputfile.getAbsolutePath())) { + + + // Because this is indexed-based dimension slicing, the dimensions are assumed to be unlimited with no names (ie. ds.dimensions == []) + // Therefore, we need to find a 'representative' variable with dimensions that we can inspect and work with + // 'lat' and 'lon' are common variable names in the datasets we work with. So try to find one of those first + // Otherwise, just find the first variable that has the same number of dimensions as was given in this.dimensions + List<String> commonVariableNames = Arrays.asList("lat", "latitude", "lon", "longitude"); + Optional<Variable> var = ds.getVariables().stream() + .filter(variable -> commonVariableNames.contains(variable.getShortName().toLowerCase()) + || variable.getDimensions().size() == this.dimensions.size()) + .findFirst(); + + assert var.isPresent() : "Could not find a variable in " + inputfile.getName() + " with " + dimensions.size() + " dimension(s)."; + + dimensionNameToLength = IntStream.range(0, this.dimensions.size()).boxed() + .collect(Collectors.toMap(dimIndex -> this.dimensionNamePrefix + dimIndex, dimIndex -> var.get().getDimension(dimIndex).getLength())); + } + + return generateTileBoundrySlices(this.dimensionNamePrefix + this.sliceByDimension, dimensionNameToLength); + + } + + List<String> namedDimensionSlicing(File inputfile) throws IOException { + Map<String, Integer> dimensionNameToLength; + try (NetcdfDataset ds = NetcdfDataset.openDataset(inputfile.getAbsolutePath())) { + + dimensionNameToLength = ds.getDimensions().stream() + .filter(dimension -> this.dimensions.contains(dimension.getShortName())) + .collect(Collectors.toMap(Dimension::getShortName, Dimension::getLength)); + } + + return generateTileBoundrySlices(this.sliceByDimension, dimensionNameToLength); + } + + List<String> generateTileBoundrySlices(String sliceByDimension, Map<String, Integer> dimensionNameToLength) { + + List<Set<String>> dimensionBounds = dimensionNameToLength.entrySet().stream() + .map(stringIntegerEntry -> { + String dimensionName = stringIntegerEntry.getKey(); + Integer lengthOfDimension = stringIntegerEntry.getValue(); + Integer stepSize = (dimensionName.equals(sliceByDimension)) ? 1 : lengthOfDimension; + + Set<String> bounds = new LinkedHashSet<>(); + for (int i = 0; i < lengthOfDimension; i += stepSize) { + bounds.add( + dimensionName + ":" + + i + ":" + + (i + stepSize >= lengthOfDimension ? lengthOfDimension : i + stepSize)); + } + return bounds; + }).collect(Collectors.toList()); + + return Sets.cartesianProduct(dimensionBounds) + .stream() + .map(tileSpecAsList -> tileSpecAsList.stream().collect(Collectors.joining(","))) + .collect(Collectors.toList()); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/datatiler/SliceFileByTilesDesired.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/datatiler/SliceFileByTilesDesired.java b/src/main/java/org/apache/sdap/ningester/datatiler/SliceFileByTilesDesired.java new file mode 100644 index 0000000..62e43bf --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/datatiler/SliceFileByTilesDesired.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sdap.ningester.datatiler; + +import com.google.common.collect.Sets; +import ucar.nc2.Dimension; +import ucar.nc2.dataset.NetcdfDataset; + +import java.io.File; +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class SliceFileByTilesDesired implements FileSlicer { + + private Integer tilesDesired; + private List<String> dimensions; + private String timeDimension; + + public void setTilesDesired(Integer desired) { + this.tilesDesired = desired; + } + + public void setDimensions(List<String> dims) { + this.dimensions = dims; + } + + @Override + public List<String> generateSlices(File inputfile) throws IOException { + + Integer timeLen = 0; + Map<String, Integer> dimensionNameToLength; + try (NetcdfDataset ds = NetcdfDataset.openDataset(inputfile.getAbsolutePath())) { + + + dimensionNameToLength = ds.getDimensions().stream() + .filter(dimension -> this.dimensions.contains(dimension.getShortName())) + .collect(Collectors.toMap(Dimension::getShortName, Dimension::getLength, + (v1, v2) -> { + throw new RuntimeException(String.format("Duplicate key for values %s and %s", v1, v2)); + }, + TreeMap::new)); + + if (this.timeDimension != null) { + timeLen = ds.getDimensions().stream() + .filter(dimension -> this.timeDimension.equals(dimension.getShortName())) + .map(Dimension::getLength) + .collect(Collectors.toList()).get(0); + } + } + + return addTimeDimension(generateChunkBoundrySlices(tilesDesired, dimensionNameToLength), timeLen); + + } + + List<String> generateChunkBoundrySlices(Integer tilesDesired, Map<String, Integer> dimensionNameToLength) { + + List<Set<String>> dimensionBounds = dimensionNameToLength.entrySet().stream() + .map(stringIntegerEntry -> { + String dimensionName = stringIntegerEntry.getKey(); + Integer lengthOfDimension = stringIntegerEntry.getValue(); + Integer stepSize = calculateStepSize(stringIntegerEntry.getValue(), tilesDesired, dimensionNameToLength.size()); + Set<String> bounds = new LinkedHashSet<>(); + for (int i = 0; i < lengthOfDimension; i += stepSize) { + bounds.add( + dimensionName + ":" + + i + ":" + + (i + stepSize >= lengthOfDimension ? lengthOfDimension : i + stepSize)); + } + return bounds; + }).collect(Collectors.toList()); + + return Sets.cartesianProduct(dimensionBounds) + .stream() + .map(tileSpecAsList -> tileSpecAsList.stream().collect(Collectors.joining(","))) + .collect(Collectors.toList()); + + } + + List<String> addTimeDimension(List<String> specs, Integer timeLen) { + + if (timeLen > 0) { + return specs.stream().map(sectionSpec -> + IntStream.range(0, timeLen) + .mapToObj(timeIndex -> this.timeDimension + ":" + timeIndex + ":" + (timeIndex + 1) + "," + sectionSpec) + .collect(Collectors.toList())) + .flatMap(List::stream) + .collect(Collectors.toList()); + } else { + return specs; + } + } + + private Integer calculateStepSize(Integer lengthOfDimension, Integer chunksDesired, Integer numberOfDimensions) { + return new Double(Math.floor(lengthOfDimension / (Math.pow(chunksDesired, (1.0 / numberOfDimensions))))).intValue(); + } + + public void setTimeDimension(String timeDimension) { + this.timeDimension = timeDimension; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/datatiler/properties/SliceFileByDimension.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/datatiler/properties/SliceFileByDimension.java b/src/main/java/org/apache/sdap/ningester/datatiler/properties/SliceFileByDimension.java new file mode 100644 index 0000000..f9d609d --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/datatiler/properties/SliceFileByDimension.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.datatiler.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.List; + +@ConfigurationProperties +@Component("sliceFileByDimensionProperties") +public class SliceFileByDimension { + + private String sliceByDimension; + private List<String> dimensions; + private String dimensionNamePrefix = ""; + + public String getSliceByDimension() { + return sliceByDimension; + } + + public void setSliceByDimension(String sliceByDimension) { + this.sliceByDimension = sliceByDimension; + } + + public List<String> getDimensions() { + return dimensions; + } + + public void setDimensions(List<String> dimensions) { + this.dimensions = dimensions; + } + + public String getDimensionNamePrefix() { + return dimensionNamePrefix; + } + + public void setDimensionNamePrefix(String dimensionNamePrefix) { + this.dimensionNamePrefix = dimensionNamePrefix; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/datatiler/properties/SliceFileByTilesDesired.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/datatiler/properties/SliceFileByTilesDesired.java b/src/main/java/org/apache/sdap/ningester/datatiler/properties/SliceFileByTilesDesired.java new file mode 100644 index 0000000..eb745d1 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/datatiler/properties/SliceFileByTilesDesired.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.datatiler.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +@ConfigurationProperties +@Component("sliceFileByTilesDesiredProperties") +public class SliceFileByTilesDesired { + private Integer tilesDesired; + private List<String> dimensions = new ArrayList<>(); + private String timeDimension; + + public Integer getTilesDesired() { + return tilesDesired; + } + + public void setTilesDesired(Integer tilesDesired) { + this.tilesDesired = tilesDesired; + } + + public List<String> getDimensions() { + return dimensions; + } + + public void setDimensions(List<String> dimensions) { + this.dimensions = dimensions; + } + + public String getTimeDimension() { + return timeDimension; + } + + public void setTimeDimension(String timeDimension) { + this.timeDimension = timeDimension; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/http/NexusTileConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/http/NexusTileConverter.java b/src/main/java/org/apache/sdap/ningester/http/NexusTileConverter.java new file mode 100644 index 0000000..2757f70 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/http/NexusTileConverter.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.http; + +import org.apache.sdap.nexusproto.NexusTile; +import org.springframework.http.HttpInputMessage; +import org.springframework.http.HttpOutputMessage; +import org.springframework.http.converter.AbstractHttpMessageConverter; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.http.converter.HttpMessageNotWritableException; + +import java.io.IOException; +import java.io.OutputStream; + +public class NexusTileConverter extends AbstractHttpMessageConverter<NexusTile> { + + @Override + protected boolean supports(Class<?> clazz) { + return NexusTile.class.isAssignableFrom(clazz); + } + + @Override + protected NexusTile readInternal(Class<? extends NexusTile> clazz, HttpInputMessage inputMessage) throws IOException, HttpMessageNotReadableException { + + return NexusTile.parseFrom(inputMessage.getBody()); + } + + @Override + protected void writeInternal(NexusTile nexusTile, HttpOutputMessage outputMessage) throws IOException, HttpMessageNotWritableException { + try { + OutputStream outputStream = outputMessage.getBody(); + nexusTile.writeTo(outputStream); + outputStream.close(); + } catch (Exception ignored) { + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/processors/AddDatasetName.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/processors/AddDatasetName.java b/src/main/java/org/apache/sdap/ningester/processors/AddDatasetName.java new file mode 100644 index 0000000..2ae13c1 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/processors/AddDatasetName.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.processors; + +import org.apache.sdap.nexusproto.NexusTile; + +public class AddDatasetName { + + String datasetName; + + public AddDatasetName(String datasetName) { + this.datasetName = datasetName; + } + + public NexusTile addDatasetName(NexusTile inputTile) { + + NexusTile.Builder outTileBuilder = NexusTile.newBuilder().mergeFrom(inputTile); + + outTileBuilder.getSummaryBuilder().setDatasetName(datasetName); + + return outTileBuilder.build(); + + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/processors/AddDayOfYearAttribute.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/processors/AddDayOfYearAttribute.java b/src/main/java/org/apache/sdap/ningester/processors/AddDayOfYearAttribute.java new file mode 100644 index 0000000..59b917a --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/processors/AddDayOfYearAttribute.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.processors; + +import org.apache.sdap.nexusproto.Attribute; +import org.apache.sdap.nexusproto.NexusTile; +import org.apache.sdap.nexusproto.TileSummary; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class AddDayOfYearAttribute { + + private Pattern regex; + + /** + * Reuqires regex that defines EXACTLY one capturing group that contains the day of year when matched with the + * granule name. + * + * @param regex Regex used to match against the granule name. Must define exactly one capturing group that captures + * the day of year. + */ + public AddDayOfYearAttribute(String regex) { + this.regex = Pattern.compile(regex); + } + + /** + * Uses regex to extract a match from the granule name that contains the day of year. + * + * @param nexusTile The tile to process + * @return The processed tile + */ + public NexusTile setDayOfYearFromGranuleName(NexusTile nexusTile) { + + String granuleName = nexusTile.getSummary().getGranule(); + Matcher granuleNameMatcher = this.regex.matcher(granuleName); + Boolean granuleNameMatched = granuleNameMatcher.find(); + + if (!granuleNameMatched) { + throw new RuntimeException("regex did not match granuleName."); + } + + if (granuleNameMatcher.groupCount() != 1) { + throw new RuntimeException("regex does not have exactly one capturing group."); + } + + if (granuleNameMatcher.group(1).length() <= 0) { + throw new RuntimeException("group does not contain match."); + } + + + String dayOfYear = granuleNameMatcher.group(1); + NexusTile.Builder newTileBuilder = NexusTile.newBuilder().mergeFrom(nexusTile); + TileSummary.Builder newTileSummaryBuilder = newTileBuilder.getSummaryBuilder(); + newTileSummaryBuilder.addGlobalAttributes( + Attribute.newBuilder() + .setName("day_of_year_i") + .addValues(dayOfYear) + .build() + ); + newTileBuilder.setSummary(newTileSummaryBuilder); + + return newTileBuilder.build(); + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/processors/AddTimeFromGranuleName.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/processors/AddTimeFromGranuleName.java b/src/main/java/org/apache/sdap/ningester/processors/AddTimeFromGranuleName.java new file mode 100644 index 0000000..c741e31 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/processors/AddTimeFromGranuleName.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sdap.ningester.processors; + +import org.apache.sdap.nexusproto.NexusTile; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; +import java.util.TimeZone; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +public class AddTimeFromGranuleName { + + + private Pattern regex; + private SimpleDateFormat dateFormat; + + public AddTimeFromGranuleName(String regex, String dateFormat) { + this.regex = Pattern.compile(regex); + this.dateFormat = new SimpleDateFormat(dateFormat, Locale.US); + this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + } + + public NexusTile setTimeFromGranuleName(NexusTile inputTile) { + + NexusTile.Builder outTileBuilder = NexusTile.newBuilder().mergeFrom(inputTile); + + switch (inputTile.getTile().getTileTypeCase()) { + case GRID_TILE: + + String granuleName = inputTile.getSummary().getGranule(); + Matcher granuleNameMatcher = this.regex.matcher(granuleName); + Boolean granuleNameMatched = granuleNameMatcher.find(); + + if (!granuleNameMatched) { + throw new RuntimeException("regex did not match granuleName."); + } + + if (granuleNameMatcher.groupCount() != 1) { + throw new RuntimeException("regex does not have exactly one capturing group."); + } + + if (granuleNameMatcher.group(1).length() <= 0) { + throw new RuntimeException("group does not contain match."); + } + + String dateTimeString = granuleNameMatcher.group(1); + Date dateTime = null; + try { + dateTime = dateFormat.parse(dateTimeString); + } catch (ParseException e) { + throw new RuntimeException(e); + } + + Long secondsSinceEpoch = (dateTime.getTime() / 1000); + + outTileBuilder.getTileBuilder().getGridTileBuilder().setTime(secondsSinceEpoch); + outTileBuilder.getSummaryBuilder().getStatsBuilder().setMinTime(secondsSinceEpoch); + outTileBuilder.getSummaryBuilder().getStatsBuilder().setMaxTime(secondsSinceEpoch); + break; + default: + throw new UnsupportedOperationException("Can only handle GridTile at this time."); + } + + return outTileBuilder.build(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/processors/CompositeItemProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/processors/CompositeItemProcessor.java b/src/main/java/org/apache/sdap/ningester/processors/CompositeItemProcessor.java new file mode 100644 index 0000000..446a592 --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/processors/CompositeItemProcessor.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.processors; + +import org.springframework.batch.item.ItemProcessor; +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; + +import java.util.ArrayList; +import java.util.List; + +public class CompositeItemProcessor<I, O> extends org.springframework.batch.item.support.CompositeItemProcessor<I, O> implements ApplicationContextAware { + + private ApplicationContext applicationContext; + + private List<String> processorBeanNames; + + public CompositeItemProcessor(List<String> processorBeanNames) { + this.processorBeanNames = processorBeanNames; + } + + @Override + @SuppressWarnings("unchecked") + public void afterPropertiesSet() { + List<ItemProcessor<I, O>> delegates = new ArrayList<>(); + for (String processorBeanName : processorBeanNames) { + delegates.add(applicationContext.getBean(processorBeanName, ItemProcessor.class)); + } + + setDelegates(delegates); + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/7a5efa30/src/main/java/org/apache/sdap/ningester/processors/GenerateTileId.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sdap/ningester/processors/GenerateTileId.java b/src/main/java/org/apache/sdap/ningester/processors/GenerateTileId.java new file mode 100644 index 0000000..1a7eb2c --- /dev/null +++ b/src/main/java/org/apache/sdap/ningester/processors/GenerateTileId.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.sdap.ningester.processors; + +import com.google.common.io.Files; +import org.apache.sdap.nexusproto.NexusTile; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.UUID; + +public class GenerateTileId { + + private String salt = ""; + + public void setSalt(String salt) { + this.salt = salt; + } + + public NexusTile addTileId(NexusTile inputTile) { + + NexusTile.Builder outTileBuilder = NexusTile.newBuilder().mergeFrom(inputTile); + String granuleFileName = inputTile.getSummary().getGranule(); + Path granulePath = null; + try { + granulePath = Paths.get(new URI(granuleFileName)); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + String granuleName = Files.getNameWithoutExtension(granulePath.getFileName().toString()); + String spec = inputTile.getSummary().getSectionSpec(); + + String tileId = UUID.nameUUIDFromBytes((granuleName + spec + salt).getBytes()).toString(); + outTileBuilder.getSummaryBuilder().setTileId(tileId); + outTileBuilder.getTileBuilder().setTileId(tileId); + + return outTileBuilder.build(); + } + + +} +
