updated reader and tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/e5e4ef73 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/e5e4ef73 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/e5e4ef73 Branch: refs/heads/master Commit: e5e4ef73e06fcd097750238fd9dda95480216158 Parents: a6e6e2a Author: Frank Greguska <[email protected]> Authored: Tue Nov 7 15:00:04 2017 -0800 Committer: Frank Greguska <[email protected]> Committed: Tue Nov 7 15:00:04 2017 -0800 ---------------------------------------------------------------------- .gitignore | 2 + .../ningester/datatiler/NetCDFItemReader.java | 41 ++++--- .../datatiler/NetCDFItemReaderTest.java | 107 +++++-------------- 3 files changed, 59 insertions(+), 91 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/e5e4ef73/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index c9f7010..72001d3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ /build/ !gradle/wrapper/gradle-wrapper.jar +/out + ### STS ### .apt_generated .classpath http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/e5e4ef73/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java index 45e68f9..35dfef1 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java @@ -5,19 +5,24 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.batch.item.*; import org.springframework.beans.factory.annotation.Autowired; +import ucar.ma2.Array; +import ucar.nc2.Variable; import ucar.nc2.dataset.NetcdfDataset; import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.IntStream; -public class NetCDFItemReader implements ItemReader<NexusContent.NexusTile>, ItemStream { +public class NetCDFItemReader implements ItemReader<String>, ItemStream { private static final Logger log = LoggerFactory.getLogger(NetCDFItemReader.class); - private static final String CURRENT_TILE_SPEC_INDEX_KEY = "current.tile.spec.index"; + static final String CURRENT_TILE_SPEC_INDEX_KEY = "current.tile.spec.index"; private List<String> tileSpecList; private Integer currentTileSpecIndex; @@ -32,7 +37,7 @@ public class NetCDFItemReader implements ItemReader<NexusContent.NexusTile>, Ite * @param fileSlicer Object responsible for slicing the NetCDF file into tiles. */ @Autowired - public NetCDFItemReader(FileSlicer fileSlicer){ + public NetCDFItemReader(FileSlicer fileSlicer) { this.fileSlicer = fileSlicer; } @@ -42,13 +47,25 @@ public class NetCDFItemReader implements ItemReader<NexusContent.NexusTile>, Ite } @Override - public NexusContent.NexusTile read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException { + public String read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException { String currentSpec = this.tileSpecList.get(this.currentTileSpecIndex); - currentSpec.split(""); - this.ds.getVariables().get(0).read(this.tileSpecList.get(this.currentTileSpecIndex++)); - - return null; + Map<String, String> dimensionToSpec = Arrays.stream(currentSpec.split(",")) + .collect(Collectors.toMap( + dimension -> dimension.split(":")[0], + dimension -> dimension.substring(dimension.indexOf(":") + 1, dimension.length()))); + + Variable varToRead = this.ds.getVariables().get(0); + String spec = varToRead.getDimensions().stream() + .map(dimension -> dimensionToSpec.get(dimension.getShortName())) + .filter(Objects::nonNull) + .collect(Collectors.joining(",")); + + spec = ":," + spec; + Array data = this.ds.getVariables().get(0).read(spec); + + this.currentTileSpecIndex++; + return data.toString(); } @Override @@ -56,16 +73,16 @@ public class NetCDFItemReader implements ItemReader<NexusContent.NexusTile>, Ite //Every time we open the file we generate the tile specs according to the given file slicer this.tileSpecList = fileSlicer.generateSlices(this.netCDFFile); - log.debug("Generated tile specifications for {}\n{}", this.netCDFFile.getName(), + log.debug("Generated tile specifications for {}\nINDEX\tTILE SPECIFICATION\n{}", this.netCDFFile.getName(), IntStream.range(0, this.tileSpecList.size()) - .mapToObj(i -> i + ": " + this.tileSpecList.get(i)) + .mapToObj(i -> i + "\t" + this.tileSpecList.get(i)) .collect(Collectors.joining("\n"))); - if(executionContext.containsKey(CURRENT_TILE_SPEC_INDEX_KEY)) { + if (!executionContext.containsKey(CURRENT_TILE_SPEC_INDEX_KEY)) { //Start at index 0 this.currentTileSpecIndex = 0; executionContext.putInt(CURRENT_TILE_SPEC_INDEX_KEY, this.currentTileSpecIndex); - }else{ + } else { //Start at index location from context this.currentTileSpecIndex = executionContext.getInt(CURRENT_TILE_SPEC_INDEX_KEY); } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/e5e4ef73/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java index 57ce44c..bd0c93e 100644 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReaderTest.java @@ -1,98 +1,47 @@ package gov.nasa.jpl.nexus.ningester.datatiler; -import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.batch.item.*; -import org.springframework.beans.factory.annotation.Autowired; -import ucar.nc2.dataset.NetcdfDataset; +import org.junit.Test; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.core.io.ClassPathResource; -import java.io.File; import java.io.IOException; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; +import java.util.Arrays; -public class NetCDFItemReaderTest implements ItemReader<NexusContent.NexusTile>, ItemStream { +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; - private static final Logger log = LoggerFactory.getLogger(NetCDFItemReaderTest.class); +public class NetCDFItemReaderTest { - private static final String CURRENT_TILE_SPEC_INDEX_KEY = "current.tile.spec.index"; + @Test + public void testOpen() throws IOException { + SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); + slicer.setTilesDesired(5184); + slicer.setDimensions(Arrays.asList("lat", "lon")); - private List<String> tileSpecList; - private Integer currentTileSpecIndex; + NetCDFItemReader reader = new NetCDFItemReader(slicer); + reader.setNetCDFFile(new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc").getFile()); - private File netCDFFile; - private NetcdfDataset ds; - private FileSlicer fileSlicer; + ExecutionContext context = new ExecutionContext(); + reader.open(context); - /** - * Constructor - * - * @param fileSlicer Object responsible for slicing the NetCDF file into tiles. - */ - @Autowired - public NetCDFItemReaderTest(FileSlicer fileSlicer){ - this.fileSlicer = fileSlicer; + assertTrue(context.containsKey(NetCDFItemReader.CURRENT_TILE_SPEC_INDEX_KEY)); } - @Autowired - public void setNetCDFFile(File netCDFFile) { - this.netCDFFile = netCDFFile; - } - - @Override - public NexusContent.NexusTile read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException { - - String currentSpec = this.tileSpecList.get(this.currentTileSpecIndex); - currentSpec.split(""); - this.ds.getVariables().get(0).read(this.tileSpecList.get(this.currentTileSpecIndex++)); - - return null; - } - - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { + @Test + public void testRead() throws Exception { + SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); + slicer.setTilesDesired(5184); + slicer.setDimensions(Arrays.asList("lat", "lon")); - //Every time we open the file we generate the tile specs according to the given file slicer - this.tileSpecList = fileSlicer.generateSlices(this.netCDFFile); - log.debug("Generated tile specifications for {}\n{}", this.netCDFFile.getName(), - IntStream.range(0, this.tileSpecList.size()) - .mapToObj(i -> i + ": " + this.tileSpecList.get(i)) - .collect(Collectors.joining("\n"))); + NetCDFItemReader reader = new NetCDFItemReader(slicer); + reader.setNetCDFFile(new ClassPathResource("granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc").getFile()); - if(executionContext.containsKey(CURRENT_TILE_SPEC_INDEX_KEY)) { - //Start at index 0 - this.currentTileSpecIndex = 0; - executionContext.putInt(CURRENT_TILE_SPEC_INDEX_KEY, this.currentTileSpecIndex); - }else{ - //Start at index location from context - this.currentTileSpecIndex = executionContext.getInt(CURRENT_TILE_SPEC_INDEX_KEY); - } - - //Open the resource - try { - this.ds = NetcdfDataset.openDataset(netCDFFile.getAbsolutePath()); - } catch (IOException e) { - throw new ItemStreamException(e); - } - - } - - @Override - public void update(ExecutionContext executionContext) throws ItemStreamException { - - executionContext.putInt(CURRENT_TILE_SPEC_INDEX_KEY, this.currentTileSpecIndex); - } + ExecutionContext context = new ExecutionContext(); + reader.open(context); - @Override - public void close() throws ItemStreamException { + String result = reader.read(); - try { - this.ds.close(); - } catch (IOException e) { - throw new ItemStreamException(e); - } + assertNotNull(result); } }
