added property files for each processor
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/5bc5aeb7 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/5bc5aeb7 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/5bc5aeb7 Branch: refs/heads/master Commit: 5bc5aeb73b891d3f97e44119c26120dfb3eed349 Parents: bc043f6 Author: Frank Greguska <[email protected]> Authored: Wed Jan 3 12:19:07 2018 -0800 Committer: Frank Greguska <[email protected]> Committed: Wed Jan 3 12:19:07 2018 -0800 ---------------------------------------------------------------------- .../ningester/configuration/AppConfig.java | 61 ++++++++++++++++---- .../properties/AddDatasetName.java | 24 ++++++++ .../properties/AddDayOfYearAttribute.java | 25 ++++++++ .../properties/AddTimeFromGranuleName.java | 33 +++++++++++ .../properties/AddTimeToSectionSpec.java | 4 +- .../properties/ApplicationProperties.java | 41 +++++++------ .../properties/GenerateTileId.java | 24 ++++++++ .../properties/PythonChainProcessor.java | 42 ++++++++++++++ .../properties/PythonChainProperties.java | 32 ---------- .../ningester/processors/AddDatasetName.java | 29 ++++++++++ .../processors/AddTimeToSectionSpec.java | 12 ++-- .../ningester/processors/GenerateTileId.java | 2 +- .../processors/PythonChainProcessor.java | 2 +- .../ningester/processors/SetDatasetName.java | 29 ---------- .../processors/TestAddDatasetName.java | 31 ++++++++++ .../processors/TestGenerateTileId.java | 4 +- .../processors/TestSetDatasetName.java | 31 ---------- src/test/resources/testjobs/AvhrrJobTest.yml | 16 +++-- 18 files changed, 304 insertions(+), 138 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java index b274220..3a8d4b5 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java @@ -7,14 +7,15 @@ package gov.nasa.jpl.nexus.ningester.configuration; import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperties; import gov.nasa.jpl.nexus.ningester.http.NexusTileConverter; -import gov.nasa.jpl.nexus.ningester.processors.AddTimeToSectionSpec; -import gov.nasa.jpl.nexus.ningester.processors.PythonChainProcessor; +import gov.nasa.jpl.nexus.ningester.processors.*; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.batch.core.configuration.annotation.JobScope; import org.springframework.batch.item.ItemProcessor; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.Resource; import org.springframework.http.MediaType; @@ -49,7 +50,7 @@ public class AppConfig { RestTemplate template = new RestTemplate(); DefaultUriTemplateHandler uriTemplateHandler = new DefaultUriTemplateHandler(); - uriTemplateHandler.setBaseUrl(applicationProperties.getNingesterpy().getBaseUrl().toString()); + uriTemplateHandler.setBaseUrl(applicationProperties.getPythonChainProcessor().getBaseUrl().toString()); template.setUriTemplateHandler(uriTemplateHandler); List<HttpMessageConverter<?>> converters = template.getMessageConverters(); @@ -59,22 +60,62 @@ public class AppConfig { return template; } + /* + * Item Processor beans defined below + */ @Bean - @JobScope - protected ItemProcessor<String, NexusContent.NexusTile> pythonChainProcessor(RestTemplate restTemplate, Resource granule) throws IOException { - PythonChainProcessor processor = new PythonChainProcessor(restTemplate); - processor.setGranule(granule); - processor.setProcessorList(applicationProperties.getPythonChainProperties().getProcessorList()); - processor.setUriPath(applicationProperties.getPythonChainProperties().getUriPath()); + @ConditionalOnProperty(prefix = "ningester.addDatasetName", name = "enabled") + protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> addDatasetName() { - return processor::sectionSpecProcessor; + AddDatasetName processor = new AddDatasetName(applicationProperties.getAddDatasetName().getDatasetName()); + return processor::addDatasetName; + } + + @Bean + @ConditionalOnProperty(prefix = "ningester.addDayOfYearAttribute", name = "enabled") + protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> addDayOfYearAttribute() { + + AddDayOfYearAttribute processor = new AddDayOfYearAttribute(applicationProperties.getAddDayOfYearAttribute().getRegex()); + return processor::setDayOfYearFromGranuleName; } @Bean + @ConditionalOnProperty(prefix = "ningester.addTimeFromGranuleName", name = "enabled") + protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> addTimeFromGranuleName() { + + AddTimeFromGranuleName processor = new AddTimeFromGranuleName(applicationProperties.getAddTimeFromGranuleName().getRegex(), applicationProperties.getAddTimeFromGranuleName().getDateFormat()); + return processor::setTimeFromGranuleName; + } + + @Bean + @JobScope + @ConditionalOnProperty(prefix = "ningester.addTimeToSectionSpec", name = "enabled") protected ItemProcessor<String, String> addTimeToSectionSpec(Resource granule) throws IOException { AddTimeToSectionSpec processor = new AddTimeToSectionSpec(applicationProperties.getAddTimeToSectionSpec().getTimeLen(), granule.getFile().getAbsolutePath()); processor.setTimeVar(applicationProperties.getAddTimeToSectionSpec().getTimeVar()); return processor::process; } + + @Bean + @ConditionalOnProperty(prefix = "ningester.generateTileId", name = "enabled") + protected ItemProcessor<NexusContent.NexusTile, NexusContent.NexusTile> generateTileId() { + + GenerateTileId processor = new GenerateTileId(); + processor.setSalt(applicationProperties.getGenerateTileId().getSalt()); + return processor::addTileId; + } + + @Bean + @JobScope + @ConditionalOnProperty(prefix = "ningester.pythonChainProcessor", name = "enabled") + protected ItemProcessor<String, NexusContent.NexusTile> pythonChainProcessor(RestTemplate restTemplate, Resource granule) throws IOException { + PythonChainProcessor processor = new PythonChainProcessor(restTemplate); + processor.setGranule(granule); + processor.setProcessorList(applicationProperties.getPythonChainProcessor().getProcessorList()); + processor.setUriPath(applicationProperties.getPythonChainProcessor().getUriPath()); + + return processor::sectionSpecProcessor; + } + } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDatasetName.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDatasetName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDatasetName.java new file mode 100644 index 0000000..8ff9230 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDatasetName.java @@ -0,0 +1,24 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.configuration.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component("addDatasetNameProperties") +public class AddDatasetName { + + private String datasetName; + + public String getDatasetName() { + return datasetName; + } + + public void setDatasetName(String datasetName) { + this.datasetName = datasetName; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDayOfYearAttribute.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDayOfYearAttribute.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDayOfYearAttribute.java new file mode 100644 index 0000000..7ec9513 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddDayOfYearAttribute.java @@ -0,0 +1,25 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.configuration.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component("addDayOfYearAttributeProperties") +public class AddDayOfYearAttribute { + + private String regex; + + + public String getRegex() { + return regex; + } + + public void setRegex(String regex) { + this.regex = regex; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeFromGranuleName.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeFromGranuleName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeFromGranuleName.java new file mode 100644 index 0000000..2995bd4 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeFromGranuleName.java @@ -0,0 +1,33 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.configuration.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component("addTimeFromGranuleNameProperties") +public class AddTimeFromGranuleName { + + private String regex; + private String dateFormat; + + public String getRegex() { + return regex; + } + + public void setRegex(String regex) { + this.regex = regex; + } + + public String getDateFormat() { + return dateFormat; + } + + public void setDateFormat(String dateFormat) { + this.dateFormat = dateFormat; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java index 118f635..5ebe198 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java @@ -9,10 +9,10 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; @ConfigurationProperties -@Component +@Component("addTimeToSectionSpecProperties") public class AddTimeToSectionSpec { - private String timeVar; + private String timeVar = "time"; private Integer timeLen; http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java index e124550..94a46af 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java @@ -9,7 +9,6 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.NestedConfigurationProperty; import org.springframework.stereotype.Component; -import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -20,19 +19,25 @@ public class ApplicationProperties { private List<String> tileProcessors = new ArrayList<>(); @NestedConfigurationProperty + private final AddDayOfYearAttribute addDayOfYearAttribute = new AddDayOfYearAttribute(); + + @NestedConfigurationProperty + private final AddTimeFromGranuleName addTimeFromGranuleName = new AddTimeFromGranuleName(); + + @NestedConfigurationProperty private final AddTimeToSectionSpec addTimeToSectionSpec = new AddTimeToSectionSpec(); @NestedConfigurationProperty - private final PythonChainProperties pythonChainProperties = new PythonChainProperties(); + private final GenerateTileId generateTileId = new GenerateTileId(); - private final Ningesterpy ningesterPy = new Ningesterpy(); + @NestedConfigurationProperty + private final PythonChainProcessor pythonChainProcessor = new PythonChainProcessor(); - public Ningesterpy getNingesterpy() { - return ningesterPy; - } + @NestedConfigurationProperty + private final AddDatasetName addDatasetName = new AddDatasetName(); - public PythonChainProperties getPythonChainProperties() { - return pythonChainProperties; + public PythonChainProcessor getPythonChainProcessor() { + return pythonChainProcessor; } public List<String> getTileProcessors() { @@ -43,18 +48,20 @@ public class ApplicationProperties { return addTimeToSectionSpec; } + public AddDayOfYearAttribute getAddDayOfYearAttribute() { + return addDayOfYearAttribute; + } - public static class Ningesterpy { - - private URL baseUrl; + public AddTimeFromGranuleName getAddTimeFromGranuleName() { + return addTimeFromGranuleName; + } - public URL getBaseUrl() { - return baseUrl; - } + public GenerateTileId getGenerateTileId() { + return generateTileId; + } - public void setBaseUrl(URL baseUrl) { - this.baseUrl = baseUrl; - } + public AddDatasetName getAddDatasetName() { + return addDatasetName; } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/GenerateTileId.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/GenerateTileId.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/GenerateTileId.java new file mode 100644 index 0000000..05caf6f --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/GenerateTileId.java @@ -0,0 +1,24 @@ +/***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.configuration.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component("generateTileIdProperties") +public class GenerateTileId { + + private String salt = ""; + + public String getSalt() { + return salt; + } + + public void setSalt(String salt) { + this.salt = salt; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProcessor.java new file mode 100644 index 0000000..ff38025 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProcessor.java @@ -0,0 +1,42 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.configuration.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +@ConfigurationProperties +@Component("pythonChainProcessorProperties") +public class PythonChainProcessor { + private URL baseUrl; + private String uriPath; + + private List<PythonProcessorModule> processorList = new ArrayList<>(); + + public String getUriPath() { + return uriPath; + } + + public void setUriPath(String uriPath) { + this.uriPath = uriPath; + } + + public List<PythonProcessorModule> getProcessorList() { + return processorList; + } + + public URL getBaseUrl() { + return baseUrl; + } + + public void setBaseUrl(URL baseUrl) { + this.baseUrl = baseUrl; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProperties.java deleted file mode 100644 index 67b598b..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProperties.java +++ /dev/null @@ -1,32 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.configuration.properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.List; - -@ConfigurationProperties -@Component -public class PythonChainProperties { - private String uriPath; - - private List<PythonProcessorModule> processorList = new ArrayList<>(); - - public String getUriPath() { - return uriPath; - } - - public void setUriPath(String uriPath) { - this.uriPath = uriPath; - } - - public List<PythonProcessorModule> getProcessorList() { - return processorList; - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDatasetName.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDatasetName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDatasetName.java new file mode 100644 index 0000000..a150936 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddDatasetName.java @@ -0,0 +1,29 @@ +/* + ****************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.processors; + +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; + + +public class AddDatasetName { + + String datasetName; + + public AddDatasetName(String datasetName) { + this.datasetName = datasetName; + } + + public NexusContent.NexusTile addDatasetName(NexusContent.NexusTile inputTile) { + + NexusContent.NexusTile.Builder outTileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(inputTile); + + outTileBuilder.getSummaryBuilder().setDatasetName(datasetName); + + return outTileBuilder.build(); + + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java index 72af070..2779101 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java @@ -17,6 +17,10 @@ public class AddTimeToSectionSpec { this.absolutefilepath = absolutefilepath; } + public void setTimeVar(String timeVar) { + this.timeVar = timeVar; + } + public String process(String sectionSpec) { StringBuilder newSectionSpec = new StringBuilder(); for (int i = 0; i < this.timeLen; i++) { @@ -31,12 +35,4 @@ public class AddTimeToSectionSpec { return newSectionSpec.toString(); } - - public String getTimeVar() { - return timeVar; - } - - public void setTimeVar(String timeVar) { - this.timeVar = timeVar; - } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java index 7bfcc0a..fba1193 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/GenerateTileId.java @@ -18,7 +18,7 @@ public class GenerateTileId { this.salt = salt; } - public NexusContent.NexusTile setTileId(NexusContent.NexusTile inputTile) { + public NexusContent.NexusTile addTileId(NexusContent.NexusTile inputTile) { NexusContent.NexusTile.Builder outTileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(inputTile); String granuleFileName = inputTile.getSummary().getGranule(); http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java index dc207a7..33e7936 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java @@ -47,7 +47,7 @@ public class PythonChainProcessor { PythonChainProcessorRequest chainProcessorRequest = new PythonChainProcessorRequest(); chainProcessorRequest.setProcessorList(processorList); - chainProcessorRequest.setInputData("time:0:1," + sectionSpec + ";file://" + granule.getAbsolutePath()); + chainProcessorRequest.setInputData(sectionSpec); HttpEntity<PythonChainProcessorRequest> requestEntity = new HttpEntity<>(chainProcessorRequest, headers); http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/main/java/gov/nasa/jpl/nexus/ningester/processors/SetDatasetName.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/SetDatasetName.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/SetDatasetName.java deleted file mode 100644 index 487b0fe..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/SetDatasetName.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - ****************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors; - -import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; - - -public class SetDatasetName { - - String datasetName; - - public SetDatasetName(String datasetName) { - this.datasetName = datasetName; - } - - public NexusContent.NexusTile addDatasetName(NexusContent.NexusTile inputTile) { - - NexusContent.NexusTile.Builder outTileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(inputTile); - - outTileBuilder.getSummaryBuilder().setDatasetName(datasetName); - - return outTileBuilder.build(); - - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddDatasetName.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddDatasetName.java b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddDatasetName.java new file mode 100644 index 0000000..4bbde27 --- /dev/null +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestAddDatasetName.java @@ -0,0 +1,31 @@ +/* + ****************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.processors; + +import org.junit.Test; +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +public class TestAddDatasetName { + + @Test + public void testDatasetName() { + + String datasetName = "testDataset"; + + NexusContent.NexusTile input = NexusContent.NexusTile.newBuilder().build(); + + AddDatasetName processor = new AddDatasetName(datasetName); + + NexusContent.NexusTile result = processor.addDatasetName(input); + + assertThat(result.getSummary().getDatasetName(), is(datasetName)); + + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java index 6747d23..d000479 100644 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestGenerateTileId.java @@ -34,7 +34,7 @@ public class TestGenerateTileId { GenerateTileId processor = new GenerateTileId(); - NexusContent.NexusTile result = processor.setTileId(inputTile); + NexusContent.NexusTile result = processor.addTileId(inputTile); assertThat(result.getSummary().getTileId(), is(expectedId)); assertThat(result.getTile().getTileId(), is(expectedId)); @@ -63,7 +63,7 @@ public class TestGenerateTileId { GenerateTileId processor = new GenerateTileId(); processor.setSalt(salt); - NexusContent.NexusTile result = processor.setTileId(inputTile); + NexusContent.NexusTile result = processor.addTileId(inputTile); assertThat(result.getSummary().getTileId(), is(expectedId)); assertThat(result.getTile().getTileId(), is(expectedId)); http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestSetDatasetName.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestSetDatasetName.java b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestSetDatasetName.java deleted file mode 100644 index 148c27a..0000000 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/TestSetDatasetName.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - ****************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors; - -import org.junit.Test; -import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; - -public class TestSetDatasetName { - - @Test - public void testDatasetName() { - - String datasetName = "testDataset"; - - NexusContent.NexusTile input = NexusContent.NexusTile.newBuilder().build(); - - SetDatasetName processor = new SetDatasetName(datasetName); - - NexusContent.NexusTile result = processor.addDatasetName(input); - - assertThat(result.getSummary().getDatasetName(), is(datasetName)); - - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/5bc5aeb7/src/test/resources/testjobs/AvhrrJobTest.yml ---------------------------------------------------------------------- diff --git a/src/test/resources/testjobs/AvhrrJobTest.yml b/src/test/resources/testjobs/AvhrrJobTest.yml index 667991c..53db84a 100644 --- a/src/test/resources/testjobs/AvhrrJobTest.yml +++ b/src/test/resources/testjobs/AvhrrJobTest.yml @@ -5,12 +5,15 @@ ningester: tile_processors: - addTimeToSectionSpec - pythonChainProcessor + - generateTileId + - addDatasetName addTimeToSectionSpec: + enabled: timeVar: time timeLen: 1 - ningesterpy: + pythonChainProcessor: + enabled: base_url: http://127.0.0.1:5000/ - python_chain_properties: uri_path: processorchain processor_list: - @@ -23,8 +26,11 @@ ningester: - name: EmptyTileFilter - - name: Subtract180Longitude - - name: KelvinToCelsius - - name: TileSummarizingProcessor \ No newline at end of file + name: TileSummarizingProcessor + generateTileId: + enabled: + addDatasetName: + enabled: + datasetName: AVHRR_OI_L4_GHRSST_NCEI \ No newline at end of file
