wired up the reader and changed it to produce NexusTiles instead of Strings
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/92889799 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/92889799 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/92889799 Branch: refs/heads/master Commit: 92889799310879bd227c3e9fb261823a75f21bcc Parents: af44673 Author: Frank Greguska <[email protected]> Authored: Thu Jan 4 12:33:48 2018 -0800 Committer: Frank Greguska <[email protected]> Committed: Thu Jan 4 12:33:48 2018 -0800 ---------------------------------------------------------------------- build.gradle | 5 +- .../ningester/configuration/AppConfig.java | 31 ++++--- .../ningester/configuration/BatchConfig.java | 31 +++---- .../properties/AddTimeToSectionSpec.java | 34 -------- .../properties/ApplicationProperties.java | 23 +++-- .../properties/SliceFileByTilesDesired.java | 44 ++++++++++ .../nexus/ningester/datatiler/FileSlicer.java | 3 +- .../ningester/datatiler/NetCDFItemReader.java | 88 +++++++++++++------- .../datatiler/SliceFileByTilesDesired.java | 39 +++++++-- .../processors/AddTimeToSectionSpec.java | 38 --------- .../processors/PythonChainProcessor.java | 20 +++-- .../datatiler/NetCDFItemReaderTest.java | 35 +++++++- .../datatiler/SliceFileByTilesDesiredTest.java | 61 ++++++++++++++ .../processors/TestAddTimeToSpatialSpec.java | 53 ------------ src/test/resources/testjobs/AvhrrJobTest.yml | 6 +- 15 files changed, 291 insertions(+), 220 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index 50fcd3a..800464b 100644 --- a/build.gradle +++ b/build.gradle @@ -33,7 +33,8 @@ repositories { } ext{ - nexusMessagesVersion = "1.0.0.RELEASE" + nexusMessagesVersion = "1.2.0.RELEASE" + protobufUtilVersion = "3.5.1" netcdfJavaVersion = '4.6.9' guavaVersion = "23.2-jre" } @@ -64,6 +65,8 @@ dependencies { compile("com.h2database:h2") compile("org.nasa.jpl.nexus:nexus-messages:$nexusMessagesVersion") + compile("com.google.protobuf:protobuf-java-util:$protobufUtilVersion") + compile("edu.ucar:cdm:${netcdfJavaVersion}") compile("com.google.guava:guava") http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java index ae1f96f..ff3c1e0 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java @@ -6,12 +6,16 @@ package gov.nasa.jpl.nexus.ningester.configuration; import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperties; +import gov.nasa.jpl.nexus.ningester.datatiler.FileSlicer; +import gov.nasa.jpl.nexus.ningester.datatiler.SliceFileByTilesDesired; import gov.nasa.jpl.nexus.ningester.http.NexusTileConverter; import gov.nasa.jpl.nexus.ningester.processors.*; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.batch.core.configuration.annotation.JobScope; import org.springframework.batch.item.ItemProcessor; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -23,6 +27,7 @@ import org.springframework.web.client.RestTemplate; import org.springframework.web.util.DefaultUriTemplateHandler; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -38,6 +43,17 @@ public class AppConfig { } @Bean + @ConditionalOnProperty(prefix = "ningester", name = "tile_slicer", havingValue = "sliceFileByTilesDesired") + @Qualifier("fileSlicer") + protected FileSlicer sliceFileByTilesDesired(){ + SliceFileByTilesDesired fileSlicer = new SliceFileByTilesDesired(); + fileSlicer.setDimensions(applicationProperties.getSliceFileByTilesDesired().getDimensions()); + fileSlicer.setTilesDesired(applicationProperties.getSliceFileByTilesDesired().getTilesDesired()); + fileSlicer.setTimeDimension(applicationProperties.getSliceFileByTilesDesired().getTimeDimension()); + return fileSlicer; + } + + @Bean protected HttpMessageConverter nexusTileConverter() { NexusTileConverter converter = new NexusTileConverter(); converter.setSupportedMediaTypes(Collections.singletonList(MediaType.APPLICATION_OCTET_STREAM)); @@ -45,6 +61,7 @@ public class AppConfig { } @Bean + @ConditionalOnProperty(prefix = "ningester.pythonChainProcessor", name = "enabled") protected RestTemplate restTemplate(HttpMessageConverter nexusTileConverter) { RestTemplate template = new RestTemplate(); @@ -87,16 +104,6 @@ public class AppConfig { } @Bean - @JobScope - @ConditionalOnProperty(prefix = "ningester.addTimeToSectionSpec", name = "enabled") - protected ItemProcessor<String, String> addTimeToSectionSpec(Resource granule) throws IOException { - - AddTimeToSectionSpec processor = new AddTimeToSectionSpec(applicationProperties.getAddTimeToSectionSpec().getTimeLen(), granule.getFile().getAbsolutePath()); - processor.setTimeVar(applicationProperties.getAddTimeToSectionSpec().getTimeVar()); - return processor::process; - } - - @Bean @ConditionalOnProperty(prefix = "ningester.generateTileId", name = "enabled") protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> generateTileId() { @@ -108,13 +115,13 @@ public class AppConfig { @Bean @JobScope @ConditionalOnProperty(prefix = "ningester.pythonChainProcessor", name = "enabled") - protected ItemProcessor<String, NexusContent.NexusTile> pythonChainProcessor(RestTemplate restTemplate, Resource granule) throws IOException { + protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> pythonChainProcessor(RestTemplate restTemplate, Resource granule) throws IOException { PythonChainProcessor processor = new PythonChainProcessor(restTemplate); processor.setGranule(granule); processor.setProcessorList(applicationProperties.getPythonChainProcessor().getProcessorList()); processor.setUriPath(applicationProperties.getPythonChainProcessor().getUriPath()); - return processor::sectionSpecProcessor; + return processor::nexusTileProcessor; } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java index fd4269e..3e32377 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java @@ -1,9 +1,9 @@ package gov.nasa.jpl.nexus.ningester.configuration; import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperties; -import gov.nasa.jpl.nexus.ningester.datatiler.SliceFileByTilesDesired; +import gov.nasa.jpl.nexus.ningester.datatiler.FileSlicer; +import gov.nasa.jpl.nexus.ningester.datatiler.NetCDFItemReader; import gov.nasa.jpl.nexus.ningester.processors.CompositeItemProcessor; -import gov.nasa.jpl.nexus.ningester.processors.PythonChainProcessor; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; @@ -13,8 +13,8 @@ import org.springframework.batch.core.configuration.annotation.JobScope; import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemStreamReader; import org.springframework.batch.item.ItemWriter; -import org.springframework.batch.item.support.ListItemReader; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; @@ -25,10 +25,6 @@ import org.springframework.context.annotation.Import; import org.springframework.core.io.Resource; import org.springframework.core.io.ResourceLoader; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - @Configuration @EnableBatchProcessing @Import(AppConfig.class) @@ -59,22 +55,15 @@ public class BatchConfig { @Bean @JobScope - protected List<String> tileSpecifications(Resource granule) throws IOException { - SliceFileByTilesDesired fileSlicer = new SliceFileByTilesDesired(); - fileSlicer.setDimensions(Arrays.asList("lat", "lon")); - fileSlicer.setTilesDesired(5184); - return fileSlicer.generateSlices(granule.getFile()); - } - - @Bean - @JobScope - protected ItemReader<String> reader(List<String> tileSpecifications) { - return new ListItemReader<>(tileSpecifications); + protected ItemStreamReader<NexusContent.NexusTile> reader(FileSlicer fileSlicer, Resource granule) { + NetCDFItemReader reader = new NetCDFItemReader(fileSlicer); + reader.setResource(granule); + return reader; } @Bean @JobScope - protected ItemProcessor<String, NexusContent.NexusTile> processor() { + protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> processor() { return new CompositeItemProcessor<>(applicationProperties.getTileProcessors()); } @@ -90,9 +79,9 @@ public class BatchConfig { @Bean @JobScope - protected Step step1(ItemReader<String> reader, ItemProcessor<String, NexusContent.NexusTile> processor, ItemWriter<NexusContent.NexusTile> writer) { + protected Step step1(ItemStreamReader<NexusContent.NexusTile> reader, ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> processor, ItemWriter<NexusContent.NexusTile> writer) { return steps.get("step1") - .<String, NexusContent.NexusTile>chunk(10) + .<NexusContent.NexusTile, NexusContent.NexusTile>chunk(10) .reader(reader) .processor(processor) .writer(writer).build(); http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java deleted file mode 100644 index 5ebe198..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java +++ /dev/null @@ -1,34 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.configuration.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@ConfigurationProperties -@Component("addTimeToSectionSpecProperties") -public class AddTimeToSectionSpec { - - private String timeVar = "time"; - - private Integer timeLen; - - public String getTimeVar() { - return timeVar; - } - - public void setTimeVar(String timeVar) { - this.timeVar = timeVar; - } - - public Integer getTimeLen() { - return timeLen; - } - - public void setTimeLen(Integer timeLen) { - this.timeLen = timeLen; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java index 94a46af..f94e53c 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java @@ -16,6 +16,11 @@ import java.util.List; @Component public class ApplicationProperties { + private String tileSlicer; + + @NestedConfigurationProperty + private final SliceFileByTilesDesired sliceFileByTilesDesired = new SliceFileByTilesDesired(); + private List<String> tileProcessors = new ArrayList<>(); @NestedConfigurationProperty @@ -25,9 +30,6 @@ public class ApplicationProperties { private final AddTimeFromGranuleName addTimeFromGranuleName = new AddTimeFromGranuleName(); @NestedConfigurationProperty - private final AddTimeToSectionSpec addTimeToSectionSpec = new AddTimeToSectionSpec(); - - @NestedConfigurationProperty private final GenerateTileId generateTileId = new GenerateTileId(); @NestedConfigurationProperty @@ -44,10 +46,6 @@ public class ApplicationProperties { return tileProcessors; } - public AddTimeToSectionSpec getAddTimeToSectionSpec() { - return addTimeToSectionSpec; - } - public AddDayOfYearAttribute getAddDayOfYearAttribute() { return addDayOfYearAttribute; } @@ -64,4 +62,15 @@ public class ApplicationProperties { return addDatasetName; } + public String getTileSlicer() { + return tileSlicer; + } + + public void setTileSlicer(String tileSlicer) { + this.tileSlicer = tileSlicer; + } + + public SliceFileByTilesDesired getSliceFileByTilesDesired() { + return sliceFileByTilesDesired; + } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByTilesDesired.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByTilesDesired.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByTilesDesired.java new file mode 100644 index 0000000..aa4d542 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByTilesDesired.java @@ -0,0 +1,44 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.configuration.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +@ConfigurationProperties +@Component("sliceFileByTilesDesiredProperties") +public class SliceFileByTilesDesired { + private Integer tilesDesired; + private List<String> dimensions = new ArrayList<>(); + private String timeDimension; + + public Integer getTilesDesired() { + return tilesDesired; + } + + public void setTilesDesired(Integer tilesDesired) { + this.tilesDesired = tilesDesired; + } + + public List<String> getDimensions() { + return dimensions; + } + + public void setDimensions(List<String> dimensions) { + this.dimensions = dimensions; + } + + public String getTimeDimension() { + return timeDimension; + } + + public void setTimeDimension(String timeDimension) { + this.timeDimension = timeDimension; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.java index 44c661c..3258119 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.java @@ -6,10 +6,11 @@ package gov.nasa.jpl.nexus.ningester.datatiler; import java.io.File; +import java.io.IOException; import java.util.List; public interface FileSlicer { - List<String> generateSlices(File inputfile); + List<String> generateSlices(File inputfile) throws IOException; } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java index 35dfef1..4a018fb 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java @@ -3,22 +3,23 @@ package gov.nasa.jpl.nexus.ningester.datatiler; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.batch.item.*; -import org.springframework.beans.factory.annotation.Autowired; -import ucar.ma2.Array; -import ucar.nc2.Variable; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.batch.item.ItemStreamException; +import org.springframework.batch.item.NonTransientResourceException; +import org.springframework.batch.item.UnexpectedInputException; +import org.springframework.batch.item.file.ResourceAwareItemReaderItemStream; +import org.springframework.core.io.Resource; import ucar.nc2.dataset.NetcdfDataset; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; -import java.util.Arrays; +import java.net.URL; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.IntStream; -public class NetCDFItemReader implements ItemReader<String>, ItemStream { +public class NetCDFItemReader implements ResourceAwareItemReaderItemStream<NexusContent.NexusTile> { private static final Logger log = LoggerFactory.getLogger(NetCDFItemReader.class); @@ -27,7 +28,7 @@ public class NetCDFItemReader implements ItemReader<String>, ItemStream { private List<String> tileSpecList; private Integer currentTileSpecIndex; - private File netCDFFile; + private Resource netCDFResource; private NetcdfDataset ds; private FileSlicer fileSlicer; @@ -36,44 +37,64 @@ public class NetCDFItemReader implements ItemReader<String>, ItemStream { * * @param fileSlicer Object responsible for slicing the NetCDF file into tiles. */ - @Autowired public NetCDFItemReader(FileSlicer fileSlicer) { this.fileSlicer = fileSlicer; } - @Autowired - public void setNetCDFFile(File netCDFFile) { - this.netCDFFile = netCDFFile; - } - @Override - public String read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException { - + public NexusContent.NexusTile read() { + if(this.currentTileSpecIndex == this.tileSpecList.size()){ + //End of stream + return null; + } String currentSpec = this.tileSpecList.get(this.currentTileSpecIndex); - Map<String, String> dimensionToSpec = Arrays.stream(currentSpec.split(",")) - .collect(Collectors.toMap( - dimension -> dimension.split(":")[0], - dimension -> dimension.substring(dimension.indexOf(":") + 1, dimension.length()))); - Variable varToRead = this.ds.getVariables().get(0); - String spec = varToRead.getDimensions().stream() - .map(dimension -> dimensionToSpec.get(dimension.getShortName())) - .filter(Objects::nonNull) - .collect(Collectors.joining(",")); + URL netCDFUrl = null; + try { + netCDFUrl = this.netCDFResource.getURL(); + } catch (IOException e) { + throw new UnexpectedInputException("Generic IOException", e); + } - spec = ":," + spec; - Array data = this.ds.getVariables().get(0).read(spec); + NexusContent.NexusTile.Builder nexusTileBuilder = NexusContent.NexusTile.newBuilder(); + nexusTileBuilder.getSummaryBuilder() + .setSectionSpec(currentSpec) + .setGranule(netCDFUrl.toString()); +// Map<String, String> dimensionToSpec = Arrays.stream(currentSpec.split(",")) +// .collect(Collectors.toMap( +// dimension -> dimension.split(":")[0], +// dimension -> dimension.substring(dimension.indexOf(":") + 1, dimension.length()))); +// +// Variable varToRead = this.ds.getVariables().get(0); +// String spec = varToRead.getDimensions().stream() +// .map(dimension -> dimensionToSpec.get(dimension.getShortName())) +// .filter(Objects::nonNull) +// .collect(Collectors.joining(",")); +// +// spec = ":," + spec; +// Array data = this.ds.getVariables().get(0).read(spec); this.currentTileSpecIndex++; - return data.toString(); + return nexusTileBuilder.build(); } @Override public void open(ExecutionContext executionContext) throws ItemStreamException { + File netCDFFile = null; + try { + netCDFFile = this.netCDFResource.getFile(); + } catch (IOException e) { + throw new ItemStreamException(e); + } + //Every time we open the file we generate the tile specs according to the given file slicer - this.tileSpecList = fileSlicer.generateSlices(this.netCDFFile); - log.debug("Generated tile specifications for {}\nINDEX\tTILE SPECIFICATION\n{}", this.netCDFFile.getName(), + try { + this.tileSpecList = fileSlicer.generateSlices(netCDFFile); + } catch (IOException e) { + throw new ItemStreamException(e); + } + log.debug("Generated tile specifications for {}\nINDEX\tTILE SPECIFICATION\n{}", netCDFFile.getName(), IntStream.range(0, this.tileSpecList.size()) .mapToObj(i -> i + "\t" + this.tileSpecList.get(i)) .collect(Collectors.joining("\n"))); @@ -112,4 +133,9 @@ public class NetCDFItemReader implements ItemReader<String>, ItemStream { } } + + @Override + public void setResource(Resource resource) { + this.netCDFResource = resource; + } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java index ba8629a..b1ca335 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java @@ -12,11 +12,13 @@ import java.io.File; import java.io.IOException; import java.util.*; import java.util.stream.Collectors; +import java.util.stream.IntStream; public class SliceFileByTilesDesired implements FileSlicer { private Integer tilesDesired; private List<String> dimensions; + private String timeDimension; public void setTilesDesired(Integer desired) { this.tilesDesired = desired; @@ -27,8 +29,9 @@ public class SliceFileByTilesDesired implements FileSlicer { } @Override - public List<String> generateSlices(File inputfile) { + public List<String> generateSlices(File inputfile) throws IOException { + Integer timeLen = 0; Map<String, Integer> dimensionNameToLength; try (NetcdfDataset ds = NetcdfDataset.openDataset(inputfile.getAbsolutePath())) { @@ -36,13 +39,20 @@ public class SliceFileByTilesDesired implements FileSlicer { dimensionNameToLength = ds.getDimensions().stream() .filter(dimension -> this.dimensions.contains(dimension.getShortName())) .collect(Collectors.toMap(Dimension::getShortName, Dimension::getLength, - (v1,v2) ->{ throw new RuntimeException(String.format("Duplicate key for values %s and %s", v1, v2));}, + (v1, v2) -> { + throw new RuntimeException(String.format("Duplicate key for values %s and %s", v1, v2)); + }, TreeMap::new)); - } catch (IOException e) { - throw new RuntimeException("Error reading netcdf file", e); + + if (this.timeDimension != null) { + timeLen = ds.getDimensions().stream() + .filter(dimension -> this.timeDimension.equals(dimension.getShortName())) + .map(Dimension::getLength) + .collect(Collectors.toList()).get(0); + } } - return generateChunkBoundrySlices(tilesDesired, dimensionNameToLength); + return addTimeDimension(generateChunkBoundrySlices(tilesDesired, dimensionNameToLength), timeLen); } @@ -54,7 +64,7 @@ public class SliceFileByTilesDesired implements FileSlicer { Integer lengthOfDimension = stringIntegerEntry.getValue(); Integer stepSize = calculateStepSize(stringIntegerEntry.getValue(), tilesDesired, dimensionNameToLength.size()); Set<String> bounds = new LinkedHashSet<>(); - for(int i = 0; i < lengthOfDimension; i += stepSize){ + for (int i = 0; i < lengthOfDimension; i += stepSize) { bounds.add( dimensionName + ":" + i + ":" + @@ -70,8 +80,25 @@ public class SliceFileByTilesDesired implements FileSlicer { } + List<String> addTimeDimension(List<String> specs, Integer timeLen) { + + if(timeLen > 0) { + return specs.stream().map(sectionSpec -> + IntStream.range(0, timeLen) + .mapToObj(timeIndex -> this.timeDimension + ":" + timeIndex + ":" + (timeIndex + 1) + "," + sectionSpec) + .collect(Collectors.toList())) + .flatMap(List::stream) + .collect(Collectors.toList()); + }else { + return specs; + } + } + private Integer calculateStepSize(Integer lengthOfDimension, Integer chunksDesired, Integer numberOfDimensions) { return new Double(Math.floor(lengthOfDimension / (Math.pow(chunksDesired, (1.0 / numberOfDimensions))))).intValue(); } + public void setTimeDimension(String timeDimension) { + this.timeDimension = timeDimension; + } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java deleted file mode 100644 index 2779101..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - ***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors; - -public class AddTimeToSectionSpec { - - private String timeVar = "time"; - private Integer timeLen; - private String absolutefilepath; - - public AddTimeToSectionSpec(Integer timeLen, String absolutefilepath) { - this.timeLen = timeLen; - this.absolutefilepath = absolutefilepath; - } - - public void setTimeVar(String timeVar) { - this.timeVar = timeVar; - } - - public String process(String sectionSpec) { - StringBuilder newSectionSpec = new StringBuilder(); - for (int i = 0; i < this.timeLen; i++) { - newSectionSpec.append(this.timeVar).append(":") - .append(i).append(":") - .append(i + 1).append(",") - .append(sectionSpec).append(";"); - } - - newSectionSpec.append("file://").append(this.absolutefilepath); - - return newSectionSpec.toString(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java index 33e7936..55eb051 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java @@ -7,6 +7,8 @@ package gov.nasa.jpl.nexus.ningester.processors; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; import gov.nasa.jpl.nexus.ningester.configuration.properties.PythonProcessorModule; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.core.io.Resource; @@ -39,7 +41,7 @@ public class PythonChainProcessor { this.granule = granule.getFile(); } - public NexusContent.NexusTile sectionSpecProcessor(String sectionSpec) { + public NexusContent.NexusTile nexusTileProcessor(NexusContent.NexusTile nexusTile) { HttpHeaders headers = new HttpHeaders(); headers.setAccept(Collections.singletonList(MediaType.APPLICATION_OCTET_STREAM)); @@ -47,7 +49,11 @@ public class PythonChainProcessor { PythonChainProcessorRequest chainProcessorRequest = new PythonChainProcessorRequest(); chainProcessorRequest.setProcessorList(processorList); - chainProcessorRequest.setInputData(sectionSpec); + try { + chainProcessorRequest.setNexusTile(JsonFormat.printer().print(nexusTile)); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } HttpEntity<PythonChainProcessorRequest> requestEntity = new HttpEntity<>(chainProcessorRequest, headers); @@ -72,7 +78,7 @@ public class PythonChainProcessor { private List<PythonProcessorModule> processorList; @JsonProperty("input_data") - private String inputData; + private String nexusTile; public List<PythonProcessorModule> getProcessorList() { return processorList; @@ -82,12 +88,12 @@ public class PythonChainProcessor { this.processorList = processorList; } - public String getInputData() { - return inputData; + public String getNexusTile() { + return nexusTile; } - public void setInputData(String inputData) { - this.inputData = inputData; + public void setNexusTile(String nexusTile) { + this.nexusTile = nexusTile; } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java index bd0c93e..57d4c89 100644 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java @@ -1,12 +1,16 @@ package gov.nasa.jpl.nexus.ningester.datatiler; import org.junit.Test; +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.batch.item.ExecutionContext; import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; import java.io.IOException; import java.util.Arrays; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -19,7 +23,7 @@ public class NetCDFItemReaderTest { slicer.setDimensions(Arrays.asList("lat", "lon")); NetCDFItemReader reader = new NetCDFItemReader(slicer); - reader.setNetCDFFile(new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc").getFile()); + reader.setResource(new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc")); ExecutionContext context = new ExecutionContext(); reader.open(context); @@ -33,15 +37,38 @@ public class NetCDFItemReaderTest { slicer.setTilesDesired(5184); slicer.setDimensions(Arrays.asList("lat", "lon")); + Resource testResource = new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc"); NetCDFItemReader reader = new NetCDFItemReader(slicer); - reader.setNetCDFFile(new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc").getFile()); + reader.setResource(testResource); ExecutionContext context = new ExecutionContext(); reader.open(context); - String result = reader.read(); + NexusContent.NexusTile result = reader.read(); - assertNotNull(result); + assertThat(result.getSummary().getSectionSpec(), is("lat:0:10,lon:0:20")); + assertThat(result.getSummary().getGranule(), is(testResource.getURL().toString())); + + } + + @Test + public void testReadWithTime() throws Exception { + SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); + slicer.setTilesDesired(5184); + slicer.setDimensions(Arrays.asList("lat", "lon")); + slicer.setTimeDimension("time"); + + Resource testResource = new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc"); + NetCDFItemReader reader = new NetCDFItemReader(slicer); + reader.setResource(testResource); + + ExecutionContext context = new ExecutionContext(); + reader.open(context); + + NexusContent.NexusTile result = reader.read(); + + assertThat(result.getSummary().getSectionSpec(), is("time:0:1,lat:0:10,lon:0:20")); + assertThat(result.getSummary().getGranule(), is(testResource.getURL().toString())); } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.java index 647bdbf..ebb2c44 100644 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.java +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.java @@ -75,6 +75,45 @@ public class SliceFileByTilesDesiredTest { } @Test + public void testAddTimeDimension() { + + SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); + slicer.setTimeDimension("time"); + + Integer tilesDesired = 4; + Integer timeLen = 3; + + Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>(); + dimensionNameToLength.put("lat", 8); + dimensionNameToLength.put("lon", 8); + + List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength); + result = slicer.addTimeDimension(result, timeLen); + + assertEquals(tilesDesired * timeLen, result.size()); + + String[] expected = new String[]{ + "time:0:1,lat:0:4,lon:0:4", + "time:1:2,lat:0:4,lon:0:4", + "time:2:3,lat:0:4,lon:0:4", + + "time:0:1,lat:0:4,lon:4:8", + "time:1:2,lat:0:4,lon:4:8", + "time:2:3,lat:0:4,lon:4:8", + + "time:0:1,lat:4:8,lon:0:4", + "time:1:2,lat:4:8,lon:0:4", + "time:2:3,lat:4:8,lon:0:4", + + "time:0:1,lat:4:8,lon:4:8", + "time:1:2,lat:4:8,lon:4:8", + "time:2:3,lat:4:8,lon:4:8"}; + assertThat(result, containsInAnyOrder(expected)); + assertThat(result, contains(expected)); + + } + + @Test public void testGenerateChunkBoundrySlicesWithMurDimensions() { SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); @@ -92,4 +131,26 @@ public class SliceFileByTilesDesiredTest { assertThat(result, hasItems("lat:0:249,lon:0:500", "lat:0:249,lon:500:1000", "lat:17928:17999,lon:35500:36000")); } + + @Test + public void testAddTimeDimensionWithMurDimensionsAndTime() { + + SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); + + Integer tilesDesired = 5184; + + Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>(); + dimensionNameToLength.put("lat", 17999); + dimensionNameToLength.put("lon", 36000); + + slicer.setTimeDimension("time"); + + List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength); + result = slicer.addTimeDimension(result, 1); + + assertEquals(tilesDesired + 72, result.size()); + + assertThat(result, hasItems("time:0:1,lat:0:249,lon:0:500", "time:0:1,lat:0:249,lon:500:1000", "time:0:1,lat:17928:17999,lon:35500:36000")); + + } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddTimeToSpatialSpec.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddTimeToSpatialSpec.java b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddTimeToSpatialSpec.java deleted file mode 100644 index 9650c93..0000000 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddTimeToSpatialSpec.java +++ /dev/null @@ -1,53 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors; - -import org.junit.Assert; -import org.junit.Test; - - -public class TestAddTimeToSpatialSpec { - - - @Test - public void testStringPayload() { - - String testSpec = "test:0:1,script:3:4"; - - AddTimeToSectionSpec processor = new AddTimeToSectionSpec(4, "afilepath"); - - String expected = "time:0:1,test:0:1,script:3:4;" + - "time:1:2,test:0:1,script:3:4;" + - "time:2:3,test:0:1,script:3:4;" + - "time:3:4,test:0:1,script:3:4;" + - "file://afilepath"; - - String result = processor.process(testSpec); - - Assert.assertEquals(expected, result); - } - - @Test - public void testTimeVarName() { - - String testSpec = "test:0:1,script:3:4"; - String timeVarName = "theTime"; - - AddTimeToSectionSpec processor = new AddTimeToSectionSpec(4, "afilepath"); - processor.setTimeVar(timeVarName); - - String expected = "theTime:0:1,test:0:1,script:3:4;" + - "theTime:1:2,test:0:1,script:3:4;" + - "theTime:2:3,test:0:1,script:3:4;" + - "theTime:3:4,test:0:1,script:3:4;" + - "file://afilepath"; - - String result = processor.process(testSpec); - - Assert.assertEquals(expected, result); - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/92889799/src/test/resources/testjobs/AvhrrJobTest.yml ---------------------------------------------------------------------- diff --git a/src/test/resources/testjobs/AvhrrJobTest.yml b/src/test/resources/testjobs/AvhrrJobTest.yml index 576c688..05ed43d 100644 --- a/src/test/resources/testjobs/AvhrrJobTest.yml +++ b/src/test/resources/testjobs/AvhrrJobTest.yml @@ -7,6 +7,7 @@ ningester: tile_slicer: sliceFileByTilesDesired sliceFileByTilesDesired: tilesDesired: 5184 + timeDimension: time dimensions: - lat - lon @@ -15,14 +16,9 @@ ningester: # Tile processors configuration ningester: tile_processors: - - addTimeToSectionSpec - pythonChainProcessor - generateTileId - addDatasetName - addTimeToSectionSpec: - enabled: - timeVar: time - timeLen: 1 pythonChainProcessor: enabled: base_url: http://127.0.0.1:5000/
