new processors and tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/fe2f7e6f Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/fe2f7e6f Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/fe2f7e6f Branch: refs/heads/master Commit: fe2f7e6f59c43ae888cd7c8f5d4f0bb555d7236b Parents: 55308ae Author: Frank Greguska <[email protected]> Authored: Tue Jan 2 15:06:18 2018 -0800 Committer: Frank Greguska <[email protected]> Committed: Tue Jan 2 15:06:18 2018 -0800 ---------------------------------------------------------------------- .../ningester/configuration/AppConfig.java | 7 +- .../properties/AddTimeToSectionSpec.java | 10 +++ .../processors/AddDayOfYearAttribute.java | 73 +++++++++++++++++ .../processors/AddTimeToSectionSpec.java | 30 +++++-- .../processors/TestAddDayOfYearAttribute.java | 83 ++++++++++++++++++++ .../processors/TestAddTimeToSpatialSpec.java | 33 ++++++++ src/test/resources/testjobs/AvhrrJobTest.yml | 1 + 7 files changed, 229 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/fe2f7e6f/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java index 04fc8c2..b274220 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java @@ -16,7 +16,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; import org.springframework.core.io.Resource; import org.springframework.http.MediaType; import org.springframework.http.converter.HttpMessageConverter; @@ -72,8 +71,10 @@ public class AppConfig { } @Bean - protected ItemProcessor<String, String> addTimeToSectionSpec() { + protected ItemProcessor<String, String> addTimeToSectionSpec(Resource granule) throws IOException { - return new AddTimeToSectionSpec(applicationProperties.getAddTimeToSectionSpec().getTimeVar())::process; + AddTimeToSectionSpec processor = new AddTimeToSectionSpec(applicationProperties.getAddTimeToSectionSpec().getTimeLen(), granule.getFile().getAbsolutePath()); + processor.setTimeVar(applicationProperties.getAddTimeToSectionSpec().getTimeVar()); + return processor::process; } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/fe2f7e6f/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java index 934406d..118f635 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java @@ -14,6 +14,8 @@ public class AddTimeToSectionSpec { private String timeVar; + private Integer timeLen; + public String getTimeVar() { return timeVar; } @@ -21,4 +23,12 @@ public class AddTimeToSectionSpec { public void setTimeVar(String timeVar) { this.timeVar = timeVar; } + + public Integer getTimeLen() { + return timeLen; + } + + public void setTimeLen(Integer timeLen) { + this.timeLen = timeLen; + } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/fe2f7e6f/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDayOfYearAttribute.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDayOfYearAttribute.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDayOfYearAttribute.java new file mode 100644 index 0000000..11ed428 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDayOfYearAttribute.java @@ -0,0 +1,73 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.processors; + +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class AddDayOfYearAttribute { + + private String regex; + + /** + * Reuqires regex that defines EXACTLY one capturing group that contains the day of year when matched with the + * granule name. + * + * @param regex Regex used to match against the granule name. Must define exactly one capturing group that captures + * the day of year. + */ + @Autowired + public AddDayOfYearAttribute(String regex) { + this.regex = regex; + } + + /** + * Uses regex to extract a match from the granule name that contains the day of year. + * + * @param nexusTile The tile to process + * @return The processed tile + */ + public NexusContent.NexusTile setDayOfYearFromGranuleName(NexusContent.NexusTile nexusTile) { + + Pattern thePattern = Pattern.compile(this.regex); + + + String granuleName = nexusTile.getSummary().getGranule(); + Matcher granuleNameMatcher = thePattern.matcher(granuleName); + Boolean granuleNameMatched = granuleNameMatcher.find(); + + if (!granuleNameMatched) { + throw new RuntimeException("regex did not match granuleName."); + } + + if (granuleNameMatcher.groupCount() != 1) { + throw new RuntimeException("regex does not have exactly one capturing group."); + } + + if (granuleNameMatcher.group(1).length() <= 0) { + throw new RuntimeException("group does not contain match."); + } + + + String dayOfYear = granuleNameMatcher.group(1); + NexusContent.NexusTile.Builder newTileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(nexusTile); + NexusContent.TileSummary.Builder newTileSummaryBuilder = newTileBuilder.getSummaryBuilder(); + newTileSummaryBuilder.addGlobalAttributes( + NexusContent.Attribute.newBuilder() + .setName("day_of_year_i") + .addValues(dayOfYear) + .build() + ); + newTileBuilder.setSummary(newTileSummaryBuilder); + + return newTileBuilder.build(); + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/fe2f7e6f/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java index a108dc9..c05f281 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java @@ -7,15 +7,35 @@ package gov.nasa.jpl.nexus.ningester.processors; public class AddTimeToSectionSpec { - private String timeVar; + private String timeVar = "time"; + private Integer timeLen; + private String absolutefilepath; - public AddTimeToSectionSpec(String timeVar){ - this.timeVar = timeVar; + public AddTimeToSectionSpec(Integer timeLen, String absolutefilepath) { + this.timeLen = timeLen; + this.absolutefilepath = absolutefilepath; } - public String process(String sectionSpec){ - return sectionSpec; + public String process(String sectionSpec) { + StringBuilder newSectionSpec = new StringBuilder(); + for (int i = 0; i < this.timeLen; i++) { + newSectionSpec.append(this.timeVar).append(":") + .append(i).append(":") + .append(i + 1).append(",") + .append(sectionSpec).append(";"); + } + + newSectionSpec.append("file://").append(this.absolutefilepath); + + return newSectionSpec.toString(); } + public String getTimeVar() { + return timeVar; + } + + public void setTimeVar(String timeVar) { + this.timeVar = timeVar; + } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/fe2f7e6f/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddDayOfYearAttribute.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddDayOfYearAttribute.java b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddDayOfYearAttribute.java new file mode 100644 index 0000000..b1b398d --- /dev/null +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddDayOfYearAttribute.java @@ -0,0 +1,83 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.processors; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.beans.HasPropertyWithValue.hasProperty; + +public class TestAddDayOfYearAttribute { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void testSuccessfulMatch() { + String regex = "^(\\d{3})"; + String granuleName = "001.L4_5day_avhrr_clim_sst_pixelMean.nc"; + NexusContent.NexusTile nexusTile = NexusContent.NexusTile.newBuilder().setSummary( + NexusContent.TileSummary.newBuilder() + .setGranule(granuleName) + .build() + ).build(); + + AddDayOfYearAttribute processor = new AddDayOfYearAttribute(regex); + + NexusContent.NexusTile result = processor.setDayOfYearFromGranuleName(nexusTile); + + assertThat(result.getSummary().getGlobalAttributesList(), contains( + hasProperty("name", is("day_of_year_i")) + )); + + String actualDayOfYear = result.getSummary().getGlobalAttributes(0).getValues(0); + assertThat(actualDayOfYear, is("001")); + } + + @Test() + public void testUnsuccessfulMatch() { + + thrown.expect(RuntimeException.class); + thrown.expectMessage("regex did not match granuleName."); + + String regex = "^(\\d{3})"; + String granuleName = "L4_5day_avhrr_clim_sst_pixelMean.nc"; + NexusContent.NexusTile nexusTile = NexusContent.NexusTile.newBuilder().setSummary( + NexusContent.TileSummary.newBuilder() + .setGranule(granuleName) + .build() + ).build(); + + AddDayOfYearAttribute processor = new AddDayOfYearAttribute(regex); + + processor.setDayOfYearFromGranuleName(nexusTile); + } + + @Test() + public void testUnsuccessfulGroupMatch() { + + thrown.expect(RuntimeException.class); + thrown.expectMessage("regex does not have exactly one capturing group."); + + String regex = "^\\d{3}"; + String granuleName = "001.L4_5day_avhrr_clim_sst_pixelMean.nc"; + NexusContent.NexusTile nexusTile = NexusContent.NexusTile.newBuilder().setSummary( + NexusContent.TileSummary.newBuilder() + .setGranule(granuleName) + .build() + ).build(); + + AddDayOfYearAttribute processor = new AddDayOfYearAttribute(regex); + + processor.setDayOfYearFromGranuleName(nexusTile); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/fe2f7e6f/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddTimeToSpatialSpec.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddTimeToSpatialSpec.java b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddTimeToSpatialSpec.java new file mode 100644 index 0000000..302ae04 --- /dev/null +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddTimeToSpatialSpec.java @@ -0,0 +1,33 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.processors; + +import org.junit.Assert; +import org.junit.Test; + + +public class TestAddTimeToSpatialSpec { + + + @Test + public void testStringPayload() { + + String testSpec = "test:0:1,script:3:4"; + + AddTimeToSectionSpec processor = new AddTimeToSectionSpec(4, "afilepath"); + + String expected = "time:0:1,test:0:1,script:3:4;" + + "time:1:2,test:0:1,script:3:4;" + + "time:2:3,test:0:1,script:3:4;" + + "time:3:4,test:0:1,script:3:4;" + + "file://afilepath"; + + String result = processor.process(testSpec); + + Assert.assertEquals(expected, result); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/fe2f7e6f/src/test/resources/testjobs/AvhrrJobTest.yml ---------------------------------------------------------------------- diff --git a/src/test/resources/testjobs/AvhrrJobTest.yml b/src/test/resources/testjobs/AvhrrJobTest.yml index c83bd20..667991c 100644 --- a/src/test/resources/testjobs/AvhrrJobTest.yml +++ b/src/test/resources/testjobs/AvhrrJobTest.yml @@ -7,6 +7,7 @@ ningester: - pythonChainProcessor addTimeToSectionSpec: timeVar: time + timeLen: 1 ningesterpy: base_url: http://127.0.0.1:5000/ python_chain_properties:
