wired in the slicebydimension properties and added smap testjob
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/4f27864e Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/4f27864e Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/4f27864e Branch: refs/heads/master Commit: 4f27864e8b0350b552688eed7c95b52899a723f7 Parents: 39ea2ef Author: Frank Greguska <[email protected]> Authored: Thu Jan 4 15:40:08 2018 -0800 Committer: Frank Greguska <[email protected]> Committed: Thu Jan 4 15:40:08 2018 -0800 ---------------------------------------------------------------------- .../ningester/configuration/AppConfig.java | 16 ++++-- .../properties/ApplicationProperties.java | 7 +++ .../properties/SliceFileByDimension.java | 44 +++++++++++++++ .../datatiler/SliceFileByDimension.java | 9 +++- .../ningester/processors/GenerateTileId.java | 15 +++++- .../processors/TestGenerateTileId.java | 34 +++++------- .../nexus/ningester/testjobs/SmapJobTest.java | 54 +++++++++++++++++++ src/test/resources/testjobs/SmapJobTest.yml | 57 ++++++++++++++++++++ 8 files changed, 208 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java index ff3c1e0..3354c40 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java @@ -7,6 +7,7 @@ package gov.nasa.jpl.nexus.ningester.configuration; import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperties; import gov.nasa.jpl.nexus.ningester.datatiler.FileSlicer; +import gov.nasa.jpl.nexus.ningester.datatiler.SliceFileByDimension; import gov.nasa.jpl.nexus.ningester.datatiler.SliceFileByTilesDesired; import gov.nasa.jpl.nexus.ningester.http.NexusTileConverter; import gov.nasa.jpl.nexus.ningester.processors.*; @@ -15,7 +16,6 @@ import org.springframework.batch.core.configuration.annotation.JobScope; import org.springframework.batch.item.ItemProcessor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -27,7 +27,6 @@ import org.springframework.web.client.RestTemplate; import org.springframework.web.util.DefaultUriTemplateHandler; import java.io.IOException; -import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -45,7 +44,7 @@ public class AppConfig { @Bean @ConditionalOnProperty(prefix = "ningester", name = "tile_slicer", havingValue = "sliceFileByTilesDesired") @Qualifier("fileSlicer") - protected FileSlicer sliceFileByTilesDesired(){ + protected FileSlicer sliceFileByTilesDesired() { SliceFileByTilesDesired fileSlicer = new SliceFileByTilesDesired(); fileSlicer.setDimensions(applicationProperties.getSliceFileByTilesDesired().getDimensions()); fileSlicer.setTilesDesired(applicationProperties.getSliceFileByTilesDesired().getTilesDesired()); @@ -54,6 +53,17 @@ public class AppConfig { } @Bean + @ConditionalOnProperty(prefix = "ningester", name = "tile_slicer", havingValue = "sliceFileByDimension") + @Qualifier("fileSlicer") + protected FileSlicer sliceFileByDimension() { + SliceFileByDimension fileSlicer = new SliceFileByDimension(); + fileSlicer.setDimensions(applicationProperties.getSliceFileByDimension().getDimensions()); + fileSlicer.setSliceByDimension(applicationProperties.getSliceFileByDimension().getSliceByDimension()); + fileSlicer.setDimensionNamePrefix(applicationProperties.getSliceFileByDimension().getDimensionNamePrefix()); + return fileSlicer; + } + + @Bean protected HttpMessageConverter nexusTileConverter() { NexusTileConverter converter = new NexusTileConverter(); converter.setSupportedMediaTypes(Collections.singletonList(MediaType.APPLICATION_OCTET_STREAM)); http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java index f94e53c..e6ea647 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java @@ -19,6 +19,9 @@ public class ApplicationProperties { private String tileSlicer; @NestedConfigurationProperty + private final SliceFileByDimension sliceFileByDimension = new SliceFileByDimension(); + + @NestedConfigurationProperty private final SliceFileByTilesDesired sliceFileByTilesDesired = new SliceFileByTilesDesired(); private List<String> tileProcessors = new ArrayList<>(); @@ -73,4 +76,8 @@ public class ApplicationProperties { public SliceFileByTilesDesired getSliceFileByTilesDesired() { return sliceFileByTilesDesired; } + + public SliceFileByDimension getSliceFileByDimension() { + return sliceFileByDimension; + } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java new file mode 100644 index 0000000..4be4187 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/SliceFileByDimension.java @@ -0,0 +1,44 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.configuration.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.List; + +@ConfigurationProperties +@Component("sliceFileByDimensionProperties") +public class SliceFileByDimension { + + private String sliceByDimension; + private List<String> dimensions; + private String dimensionNamePrefix = ""; + + public String getSliceByDimension() { + return sliceByDimension; + } + + public void setSliceByDimension(String sliceByDimension) { + this.sliceByDimension = sliceByDimension; + } + + public List<String> getDimensions() { + return dimensions; + } + + public void setDimensions(List<String> dimensions) { + this.dimensions = dimensions; + } + + public String getDimensionNamePrefix() { + return dimensionNamePrefix; + } + + public void setDimensionNamePrefix(String dimensionNamePrefix) { + this.dimensionNamePrefix = dimensionNamePrefix; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java index 003edaa..0a5b594 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByDimension.java @@ -5,6 +5,7 @@ * ****************************************************************************/ package gov.nasa.jpl.nexus.ningester.datatiler; +import com.google.common.base.Strings; import com.google.common.collect.Sets; import ucar.nc2.Dimension; import ucar.nc2.Variable; @@ -16,7 +17,7 @@ import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; -class SliceFileByDimension implements FileSlicer { +public class SliceFileByDimension implements FileSlicer { private String sliceByDimension; private List<String> dimensions; @@ -47,6 +48,12 @@ class SliceFileByDimension implements FileSlicer { } List<String> indexedDimensionSlicing(File inputfile) throws IOException { + + // This is sort of a hack to help the python netcdf library. When you try to get dimensions by name and they are unnamed, the + // python library uses 'phony_dim_' then the index of the dimension as the dimension name. Weird, I know. + if(Strings.isNullOrEmpty(this.dimensionNamePrefix)){ + this.dimensionNamePrefix = "phony_dim_"; + } Map<String, Integer> dimensionNameToLength; try (NetcdfDataset ds = NetcdfDataset.openDataset(inputfile.getAbsolutePath())) { http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java index fba1193..6d00a99 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java @@ -6,8 +6,13 @@ package gov.nasa.jpl.nexus.ningester.processors; +import com.google.common.io.Files; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.UUID; public class GenerateTileId { @@ -18,11 +23,17 @@ public class GenerateTileId { this.salt = salt; } - public NexusContent.NexusTile addTileId(NexusContent.NexusTile inputTile) { + public NexusContent.NexusTile addTileId(NexusContent.NexusTile inputTile){ NexusContent.NexusTile.Builder outTileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(inputTile); String granuleFileName = inputTile.getSummary().getGranule(); - String granuleName = granuleFileName.substring(0, granuleFileName.length() - 3); + Path granulePath = null; + try { + granulePath = Paths.get(new URI(granuleFileName)); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + String granuleName = Files.getNameWithoutExtension(granulePath.getFileName().toString()); String spec = inputTile.getSummary().getSectionSpec(); String tileId = UUID.nameUUIDFromBytes((granuleName + spec + salt).getBytes()).toString(); http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java index d000479..ea18c3a 100644 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java @@ -17,24 +17,19 @@ public class TestGenerateTileId { @Test public void testGenerateId() { - String granuleFileName = "19960421120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc"; + String granuleFileName = "file:/path/to/some/file/19960421120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc"; String sectionSpec = "time:0:1,lat:140:160,lon:640:680"; String expectedId = "c031a9c4-9e1d-32e9-9d5c-d2497ce74920"; - NexusContent.NexusTile inputTile = NexusContent.NexusTile.newBuilder() - .setSummary( - NexusContent.TileSummary.newBuilder() - .setGranule(granuleFileName) - .setSectionSpec(sectionSpec) - .build() - ) - .setTile(NexusContent.TileData.newBuilder()) - .build(); + NexusContent.NexusTile.Builder inputTileBuilder = NexusContent.NexusTile.newBuilder(); + inputTileBuilder.getSummaryBuilder().setGranule(granuleFileName); + inputTileBuilder.getSummaryBuilder().setSectionSpec(sectionSpec); + inputTileBuilder.setTile(NexusContent.TileData.newBuilder()); GenerateTileId processor = new GenerateTileId(); - NexusContent.NexusTile result = processor.addTileId(inputTile); + NexusContent.NexusTile result = processor.addTileId(inputTileBuilder.build()); assertThat(result.getSummary().getTileId(), is(expectedId)); assertThat(result.getTile().getTileId(), is(expectedId)); @@ -44,26 +39,21 @@ public class TestGenerateTileId { @Test public void testGenerateIdWithSalt() { - String granuleFileName = "CCMP_Wind_Analysis_19990928_V02.0_L3.0_RSS.nc"; + String granuleFileName = "file:/path/to/some/file/CCMP_Wind_Analysis_19990928_V02.0_L3.0_RSS.nc"; String sectionSpec = "time:3:4,longitude:174:261,latitude:152:190"; String salt = "wind_u"; String expectedId = "48da50ef-e92c-3562-89f9-470561a06482"; - NexusContent.NexusTile inputTile = NexusContent.NexusTile.newBuilder() - .setSummary( - NexusContent.TileSummary.newBuilder() - .setGranule(granuleFileName) - .setSectionSpec(sectionSpec) - .build() - ) - .setTile(NexusContent.TileData.newBuilder()) - .build(); + NexusContent.NexusTile.Builder inputTileBuilder = NexusContent.NexusTile.newBuilder(); + inputTileBuilder.getSummaryBuilder().setGranule(granuleFileName); + inputTileBuilder.getSummaryBuilder().setSectionSpec(sectionSpec); + inputTileBuilder.setTile(NexusContent.TileData.newBuilder()); GenerateTileId processor = new GenerateTileId(); processor.setSalt(salt); - NexusContent.NexusTile result = processor.addTileId(inputTile); + NexusContent.NexusTile result = processor.addTileId(inputTileBuilder.build()); assertThat(result.getSummary().getTileId(), is(expectedId)); assertThat(result.getTile().getTileId(), is(expectedId)); http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java new file mode 100644 index 0000000..d99dc2b --- /dev/null +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java @@ -0,0 +1,54 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.testjobs; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +@TestPropertySource(properties = { "spring.config.location = classpath:testjobs/SmapJobTest.yml" }) +@ActiveProfiles({"test"}) +public class SmapJobTest { + + @TestConfiguration + static class NingesterApplicationTestsConfig { + + @Bean + JobLauncherTestUtils jobLauncherTestUtils() { + return new JobLauncherTestUtils(); + } + + } + + @Autowired + JobLauncherTestUtils jobLauncherTestUtils; + + + @Test + public void testJobCompletes() throws Exception { + + JobParameters jobParameters = new JobParametersBuilder() + .addString("granule", "classpath:granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5") + .toJobParameters(); + + JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); + + Assert.assertEquals("COMPLETED", jobExecution.getExitStatus().getExitCode()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/4f27864e/src/test/resources/testjobs/SmapJobTest.yml ---------------------------------------------------------------------- diff --git a/src/test/resources/testjobs/SmapJobTest.yml b/src/test/resources/testjobs/SmapJobTest.yml new file mode 100644 index 0000000..e8fda0d --- /dev/null +++ b/src/test/resources/testjobs/SmapJobTest.yml @@ -0,0 +1,57 @@ +spring: + profiles: test + +--- +# Tile Slicer Config +ningester: + tile_slicer: sliceFileByDimension + sliceFileByDimension: + sliceByDimension: 1 + dimensionNamePrefix: phony_dim_ + dimensions: + - 0 + - 1 + +--- +# Tile processors configuration +ningester: + tile_processors: + - pythonChainProcessor + - generateTileId + - addDatasetName + pythonChainProcessor: + enabled: + base_url: http://127.0.0.1:5000/ + uri_path: processorchain + processor_list: + - + name: SwathReadingProcessor + config: + latitude: lat + longitude: lon + time: row_time + variable_to_read: smap_sss + glblattr_day: REV_START_TIME + glblattr_day_format: '%Y-%jT%H:%M:%S.%f' + - + name: EmptyTileFilter + - + name: TileSummarizingProcessor + generateTileId: + enabled: + addDatasetName: + enabled: + datasetName: SMAP_L2B_SSS +--- +# Tile writer configuration +ningester: + tile_writer: + data_store: cassandraStore + metadata_store: solrStore + cassandraStore: + cassandraContactPoints: 127.0.0.1 + cassandraKeyspace: nexustiles + cassandraPort: 9042 + solrStore: + solrUrl: http://127.0.0.1:8983 + solrCollection: nexustiles \ No newline at end of file
