more progress on batch job
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/55308aec Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/55308aec Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/55308aec Branch: refs/heads/master Commit: 55308aec2cbba620085354a809bf76563403b258 Parents: e5e4ef7 Author: Frank Greguska <[email protected]> Authored: Wed Dec 27 16:32:16 2017 -0600 Committer: Frank Greguska <[email protected]> Committed: Wed Dec 27 16:32:16 2017 -0600 ---------------------------------------------------------------------- .gitignore | 2 + build.gradle | 23 +- .../ningester/configuration/AppConfig.java | 79 ++++++ .../ningester/configuration/BatchConfig.java | 104 +++++--- .../configuration/DatasourceConfig.java | 22 ++ .../configuration/DatasourceConfiguration.java | 22 -- .../properties/AddTimeToSectionSpec.java | 24 ++ .../properties/ApplicationProperties.java | 60 +++++ .../properties/PythonChainProperties.java | 32 +++ .../properties/PythonProcessorModule.java | 27 +++ .../ningester/http/NexusTileConverter.java | 40 ++++ .../processors/AddTimeToSectionSpec.java | 21 ++ .../processors/CompositeItemProcessor.java | 41 ++++ .../processors/PythonChainProcessor.java | 93 ++++++++ .../ningester/processors/PythonProcessor.java | 33 --- src/python/test/pymodule.py | 9 - src/python/tilereadingprocessor.py | 239 ------------------- .../ningester/NingesterApplicationTests.java | 44 +--- .../processors/PythonProcessorTest.java | 40 ---- .../nexus/ningester/testjobs/AvhrrJobTest.java | 54 +++++ src/test/resources/application.properties | 2 +- src/test/resources/testjobs/AvhrrJobTest.yml | 29 +++ 22 files changed, 609 insertions(+), 431 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 72001d3..d5a17c0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ /out +.DS_Store + ### STS ### .apt_generated .classpath http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index be191f1..50fcd3a 100644 --- a/build.gradle +++ b/build.gradle @@ -1,18 +1,24 @@ buildscript { ext { - springBootVersion = '1.5.7.RELEASE' + springBootVersion = '1.5.9.RELEASE' } repositories { mavenCentral() + maven { url 'http://repo.spring.io/plugins-release' } } dependencies { classpath("org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}") + classpath 'io.spring.gradle:propdeps-plugin:0.0.9.RELEASE' } } apply plugin: 'java' apply plugin: 'idea' apply plugin: 'org.springframework.boot' +apply plugin: 'propdeps' +apply plugin: 'propdeps-maven' +apply plugin: 'propdeps-idea' +apply plugin: 'propdeps-eclipse' group = 'gov.nasa.jpl.nexus.ningester' version = '0.0.1-SNAPSHOT' @@ -30,7 +36,6 @@ ext{ nexusMessagesVersion = "1.0.0.RELEASE" netcdfJavaVersion = '4.6.9' guavaVersion = "23.2-jre" - jythonVersion = "2.7.1" } sourceSets{ @@ -52,14 +57,18 @@ sourceSets{ } dependencies { - compile('org.springframework.boot:spring-boot-starter-batch') + optional("org.springframework.boot:spring-boot-configuration-processor") + + compile("org.springframework.boot:spring-boot-starter-batch") + compile("org.springframework:spring-web") + compile("com.h2database:h2") compile("org.nasa.jpl.nexus:nexus-messages:$nexusMessagesVersion") compile("edu.ucar:cdm:${netcdfJavaVersion}") - compile group: 'com.google.guava', name: 'guava' - // https://mvnrepository.com/artifact/org.python/jython-standalone - compile group: 'org.python', name: 'jython-standalone', version: jythonVersion + compile("com.google.guava:guava") testCompile('org.springframework.boot:spring-boot-starter-test') testCompile('org.springframework.batch:spring-batch-test') -} \ No newline at end of file +} + +compileJava.dependsOn(processResources) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java new file mode 100644 index 0000000..04fc8c2 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/AppConfig.java @@ -0,0 +1,79 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.configuration; + +import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperties; +import gov.nasa.jpl.nexus.ningester.http.NexusTileConverter; +import gov.nasa.jpl.nexus.ningester.processors.AddTimeToSectionSpec; +import gov.nasa.jpl.nexus.ningester.processors.PythonChainProcessor; +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; +import org.springframework.batch.core.configuration.annotation.JobScope; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; +import org.springframework.core.io.Resource; +import org.springframework.http.MediaType; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.DefaultUriTemplateHandler; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +@Configuration +@EnableConfigurationProperties({ApplicationProperties.class}) +public class AppConfig { + + private final ApplicationProperties applicationProperties; + + @Autowired + public AppConfig(ApplicationProperties applicationProperties) { + this.applicationProperties = applicationProperties; + } + + @Bean + protected HttpMessageConverter nexusTileConverter() { + NexusTileConverter converter = new NexusTileConverter(); + converter.setSupportedMediaTypes(Collections.singletonList(MediaType.APPLICATION_OCTET_STREAM)); + return converter; + } + + @Bean + protected RestTemplate restTemplate(HttpMessageConverter nexusTileConverter) { + RestTemplate template = new RestTemplate(); + + DefaultUriTemplateHandler uriTemplateHandler = new DefaultUriTemplateHandler(); + uriTemplateHandler.setBaseUrl(applicationProperties.getNingesterpy().getBaseUrl().toString()); + template.setUriTemplateHandler(uriTemplateHandler); + + List<HttpMessageConverter<?>> converters = template.getMessageConverters(); + converters.add(nexusTileConverter); + template.setMessageConverters(converters); + + return template; + } + + @Bean + @JobScope + protected ItemProcessor<String, NexusContent.NexusTile> pythonChainProcessor(RestTemplate restTemplate, Resource granule) throws IOException { + PythonChainProcessor processor = new PythonChainProcessor(restTemplate); + processor.setGranule(granule); + processor.setProcessorList(applicationProperties.getPythonChainProperties().getProcessorList()); + processor.setUriPath(applicationProperties.getPythonChainProperties().getUriPath()); + + return processor::sectionSpecProcessor; + } + + @Bean + protected ItemProcessor<String, String> addTimeToSectionSpec() { + + return new AddTimeToSectionSpec(applicationProperties.getAddTimeToSectionSpec().getTimeVar())::process; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java index 7febcdf..fd4269e 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java @@ -1,22 +1,27 @@ package gov.nasa.jpl.nexus.ningester.configuration; -import gov.nasa.jpl.nexus.ningester.datatiler.FileSlicer; +import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperties; import gov.nasa.jpl.nexus.ningester.datatiler.SliceFileByTilesDesired; +import gov.nasa.jpl.nexus.ningester.processors.CompositeItemProcessor; +import gov.nasa.jpl.nexus.ningester.processors.PythonChainProcessor; +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; import org.springframework.batch.core.configuration.annotation.JobScope; import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.ItemWriter; -import org.springframework.batch.item.adapter.ItemWriterAdapter; import org.springframework.batch.item.support.ListItemReader; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; import org.springframework.core.io.Resource; import org.springframework.core.io.ResourceLoader; @@ -26,50 +31,71 @@ import java.util.List; @Configuration @EnableBatchProcessing +@Import(AppConfig.class) public class BatchConfig { - @Autowired - private JobBuilderFactory jobs; + @Autowired + protected JobBuilderFactory jobs; - @Autowired - private StepBuilderFactory steps; + @Autowired + protected StepBuilderFactory steps; - @Bean - public Job job(@Qualifier("step1") Step step1) { - return jobs.get("testNc4").start(step1).build(); - } + @Autowired + protected ApplicationProperties applicationProperties; - @Bean - @JobScope - protected Resource granule(ResourceLoader resourceLoader, @Value("#{jobParameters['granule']}") String granuleLocation){ - return resourceLoader.getResource(granuleLocation); - } + @Autowired + protected ApplicationContext context; - @Bean - @JobScope - protected List<String> tileSpecifications(Resource granule) throws IOException { - SliceFileByTilesDesired fileSlicer = new SliceFileByTilesDesired(); - fileSlicer.setDimensions(Arrays.asList("lat", "lon")); - fileSlicer.setTilesDesired(5184); - return fileSlicer.generateSlices(granule.getFile()); - } + @Bean + public Job job(@Qualifier("step1") Step step1) { + return jobs.get("testNc4").start(step1).build(); + } - @Bean - @JobScope - protected ItemReader<String> reader(List<String> tileSpecifications) { - return new ListItemReader<>(tileSpecifications); - } + @Bean + @JobScope + protected Resource granule(ResourceLoader resourceLoader, @Value("#{jobParameters['granule']}") String granuleLocation) { + return resourceLoader.getResource(granuleLocation); + } - @Bean - protected ItemWriter<String> writer() { - ItemWriterAdapter<String> writer = new ItemWriterAdapter<>(); - writer.setTargetMethod("println"); - writer.setTargetObject(System.out); - return writer; - } + @Bean + @JobScope + protected List<String> tileSpecifications(Resource granule) throws IOException { + SliceFileByTilesDesired fileSlicer = new SliceFileByTilesDesired(); + fileSlicer.setDimensions(Arrays.asList("lat", "lon")); + fileSlicer.setTilesDesired(5184); + return fileSlicer.generateSlices(granule.getFile()); + } + + @Bean + @JobScope + protected ItemReader<String> reader(List<String> tileSpecifications) { + return new ListItemReader<>(tileSpecifications); + } + + @Bean + @JobScope + protected ItemProcessor<String, NexusContent.NexusTile> processor() { + return new CompositeItemProcessor<>(applicationProperties.getTileProcessors()); + } + + @Bean + @JobScope + protected ItemWriter<NexusContent.NexusTile> writer() { + return items -> { + for (NexusContent.NexusTile item : items) { + System.out.println("Got tile"); + } + }; + } + + @Bean + @JobScope + protected Step step1(ItemReader<String> reader, ItemProcessor<String, NexusContent.NexusTile> processor, ItemWriter<NexusContent.NexusTile> writer) { + return steps.get("step1") + .<String, NexusContent.NexusTile>chunk(10) + .reader(reader) + .processor(processor) + .writer(writer).build(); + } - @Bean - protected Step step1(ItemReader<String> reader, ItemWriter<String> writer) { - return steps.get("step1").<String, String>chunk(10).reader(reader).writer(writer).build(); - } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java new file mode 100644 index 0000000..4868673 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfig.java @@ -0,0 +1,22 @@ +package gov.nasa.jpl.nexus.ningester.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class DatasourceConfig { + + @Bean + @Profile("embedded") + public DataSource dataSource(){ + EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder(); + return builder + .setType(EmbeddedDatabaseType.H2) + .build(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfiguration.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfiguration.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfiguration.java deleted file mode 100644 index eb3e498..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package gov.nasa.jpl.nexus.ningester.configuration; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; - -import javax.sql.DataSource; - -@Configuration -public class DatasourceConfiguration { - - @Bean - @Profile("embedded") - public DataSource dataSource(){ - EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder(); - return builder - .setType(EmbeddedDatabaseType.H2) - .build(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java new file mode 100644 index 0000000..934406d --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/AddTimeToSectionSpec.java @@ -0,0 +1,24 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.configuration.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@ConfigurationProperties +@Component +public class AddTimeToSectionSpec { + + private String timeVar; + + public String getTimeVar() { + return timeVar; + } + + public void setTimeVar(String timeVar) { + this.timeVar = timeVar; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java new file mode 100644 index 0000000..e124550 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/ApplicationProperties.java @@ -0,0 +1,60 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.configuration.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.NestedConfigurationProperty; +import org.springframework.stereotype.Component; + +import java.net.URL; +import java.util.ArrayList; +import java.util.List; + +@ConfigurationProperties("ningester") +@Component +public class ApplicationProperties { + + private List<String> tileProcessors = new ArrayList<>(); + + @NestedConfigurationProperty + private final AddTimeToSectionSpec addTimeToSectionSpec = new AddTimeToSectionSpec(); + + @NestedConfigurationProperty + private final PythonChainProperties pythonChainProperties = new PythonChainProperties(); + + private final Ningesterpy ningesterPy = new Ningesterpy(); + + public Ningesterpy getNingesterpy() { + return ningesterPy; + } + + public PythonChainProperties getPythonChainProperties() { + return pythonChainProperties; + } + + public List<String> getTileProcessors() { + return tileProcessors; + } + + public AddTimeToSectionSpec getAddTimeToSectionSpec() { + return addTimeToSectionSpec; + } + + + public static class Ningesterpy { + + private URL baseUrl; + + public URL getBaseUrl() { + return baseUrl; + } + + public void setBaseUrl(URL baseUrl) { + this.baseUrl = baseUrl; + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProperties.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProperties.java new file mode 100644 index 0000000..67b598b --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonChainProperties.java @@ -0,0 +1,32 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.configuration.properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +@ConfigurationProperties +@Component +public class PythonChainProperties { + private String uriPath; + + private List<PythonProcessorModule> processorList = new ArrayList<>(); + + public String getUriPath() { + return uriPath; + } + + public void setUriPath(String uriPath) { + this.uriPath = uriPath; + } + + public List<PythonProcessorModule> getProcessorList() { + return processorList; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonProcessorModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonProcessorModule.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonProcessorModule.java new file mode 100644 index 0000000..287e449 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/properties/PythonProcessorModule.java @@ -0,0 +1,27 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.configuration.properties; + +import java.util.HashMap; +import java.util.Map; + +public class PythonProcessorModule { + + private String name; + private Map<String, String> config = new HashMap<>(); + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Map<String, String> getConfig() { + return config; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/main/java/gov/nasa/jpl/nexus/ningester/http/NexusTileConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/http/NexusTileConverter.java b/src/main/java/gov/nasa/jpl/nexus/ningester/http/NexusTileConverter.java new file mode 100644 index 0000000..5f1245c --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/http/NexusTileConverter.java @@ -0,0 +1,40 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.http; + +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; +import org.springframework.http.HttpInputMessage; +import org.springframework.http.HttpOutputMessage; +import org.springframework.http.converter.AbstractHttpMessageConverter; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.http.converter.HttpMessageNotWritableException; + +import java.io.IOException; +import java.io.OutputStream; + +public class NexusTileConverter extends AbstractHttpMessageConverter<NexusContent.NexusTile> { + + @Override + protected boolean supports(Class<?> clazz) { + return NexusContent.NexusTile.class.isAssignableFrom(clazz); + } + + @Override + protected NexusContent.NexusTile readInternal(Class<? extends NexusContent.NexusTile> clazz, HttpInputMessage inputMessage) throws IOException, HttpMessageNotReadableException { + + return NexusContent.NexusTile.parseFrom(inputMessage.getBody()); + } + + @Override + protected void writeInternal(NexusContent.NexusTile nexusTile, HttpOutputMessage outputMessage) throws IOException, HttpMessageNotWritableException { + try { + OutputStream outputStream = outputMessage.getBody(); + nexusTile.writeTo(outputStream); + outputStream.close(); + } catch (Exception ignored) { + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java new file mode 100644 index 0000000..a108dc9 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/AddTimeToSectionSpec.java @@ -0,0 +1,21 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.processors; + +public class AddTimeToSectionSpec { + + private String timeVar; + + public AddTimeToSectionSpec(String timeVar){ + this.timeVar = timeVar; + } + + public String process(String sectionSpec){ + return sectionSpec; + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/main/java/gov/nasa/jpl/nexus/ningester/processors/CompositeItemProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/CompositeItemProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/CompositeItemProcessor.java new file mode 100644 index 0000000..463f6b9 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/CompositeItemProcessor.java @@ -0,0 +1,41 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.processors; + +import org.springframework.batch.item.ItemProcessor; +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; + +import java.util.ArrayList; +import java.util.List; + +public class CompositeItemProcessor<I, O> extends org.springframework.batch.item.support.CompositeItemProcessor<I, O> implements ApplicationContextAware { + + private ApplicationContext applicationContext; + + private List<String> processorBeanNames; + + public CompositeItemProcessor(List<String> processorBeanNames) { + this.processorBeanNames = processorBeanNames; + } + + @Override + @SuppressWarnings("unchecked") + public void afterPropertiesSet() { + List<ItemProcessor<I, O>> delegates = new ArrayList<>(); + for (String processorBeanName : processorBeanNames) { + delegates.add(applicationContext.getBean(processorBeanName, ItemProcessor.class)); + } + + setDelegates(delegates); + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java new file mode 100644 index 0000000..16ba33f --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonChainProcessor.java @@ -0,0 +1,93 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.processors; + +import com.fasterxml.jackson.annotation.JsonProperty; +import gov.nasa.jpl.nexus.ningester.configuration.properties.PythonProcessorModule; +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.web.client.RestTemplate; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class PythonChainProcessor { + + private RestTemplate restTemplate; + + private List<PythonProcessorModule> processorList; + + private String uriPath; + + private File granule; + + public PythonChainProcessor(RestTemplate restTemplate) { + this.restTemplate = restTemplate; + } + + public void setGranule(Resource granule) throws IOException { + this.granule = granule.getFile(); + } + + public NexusContent.NexusTile sectionSpecProcessor(String sectionSpec) { + + HttpHeaders headers = new HttpHeaders(); + headers.setAccept(Collections.singletonList(MediaType.APPLICATION_OCTET_STREAM)); + headers.setContentType(MediaType.APPLICATION_JSON); + + PythonChainProcessorRequest chainProcessorRequest = new PythonChainProcessorRequest(); + chainProcessorRequest.setProcessorList(processorList); + chainProcessorRequest.setInputData("time:0:1," + sectionSpec + ";file://" + granule.getAbsolutePath()); + + HttpEntity<PythonChainProcessorRequest> requestEntity = new HttpEntity<>(chainProcessorRequest, headers); + + return restTemplate.exchange( + uriPath, + HttpMethod.POST, + requestEntity, + NexusContent.NexusTile.class).getBody(); + } + + public void setProcessorList(List<PythonProcessorModule> processorList) { + this.processorList = processorList; + } + + public void setUriPath(String uriPath) { + this.uriPath = uriPath; + } + + public class PythonChainProcessorRequest { + + @JsonProperty("processor_list") + private List<PythonProcessorModule> processorList; + + @JsonProperty("input_data") + private String inputData; + + public List<PythonProcessorModule> getProcessorList() { + return processorList; + } + + public void setProcessorList(List<PythonProcessorModule> processorList) { + this.processorList = processorList; + } + + public String getInputData() { + return inputData; + } + + public void setInputData(String inputData) { + this.inputData = inputData; + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonProcessor.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonProcessor.java b/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonProcessor.java deleted file mode 100644 index b7b0baa..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/processors/PythonProcessor.java +++ /dev/null @@ -1,33 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors; - -import org.python.core.PyFunction; -import org.python.core.PyString; -import org.python.util.PythonInterpreter; -import org.springframework.beans.factory.annotation.Autowired; - -public class PythonProcessor { - - private PythonInterpreter interpreter; - private PyFunction pythonFunction; - - @Autowired - public PythonProcessor(PythonInterpreter interpreter, String pythonModule, String pythonMethod) { - this.interpreter = interpreter; - - String importedAs = pythonModule + Character.toUpperCase(pythonMethod.charAt(0)) + pythonMethod.substring(1); - interpreter.exec("from " + pythonModule + " import " + pythonMethod + " as " + importedAs); - this.pythonFunction = (PyFunction) this.interpreter.get(importedAs); - } - - - public String processWithPython(String item) { - - return pythonFunction.__call__(new PyString(item)).asString(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/python/test/pymodule.py ---------------------------------------------------------------------- diff --git a/src/python/test/pymodule.py b/src/python/test/pymodule.py deleted file mode 100644 index 8d61e27..0000000 --- a/src/python/test/pymodule.py +++ /dev/null @@ -1,9 +0,0 @@ - -def capitalize(value): - - return value.upper() - - -def square(value): - import numpy - return str(numpy.square(int(value))) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/python/tilereadingprocessor.py ---------------------------------------------------------------------- diff --git a/src/python/tilereadingprocessor.py b/src/python/tilereadingprocessor.py deleted file mode 100644 index 51474fd..0000000 --- a/src/python/tilereadingprocessor.py +++ /dev/null @@ -1,239 +0,0 @@ -""" -Copyright (c) 2016 Jet Propulsion Laboratory, -California Institute of Technology. All rights reserved -""" -import datetime -import urllib2 -from collections import OrderedDict -from contextlib import contextmanager -from os import environ, sep, path, remove - -import nexusproto.NexusContent_pb2 as nexusproto -import numpy -from netCDF4 import Dataset, num2date -from nexusproto.serialization import to_shaped_array, to_metadata -from pytz import timezone - -from springxd.tcpstream import LengthHeaderTcpProcessor, start_server - -EPOCH = timezone('UTC').localize(datetime.datetime(1970, 1, 1)) -# Required variables for all reader types -variable_to_read = environ['VARIABLE'] -latitude = environ['LATITUDE'] -longitude = environ['LONGITUDE'] -reader_type = environ['READER'] - -# Optional variables for all reader types -try: - temp_dir = environ['TEMP_DIR'] -except KeyError: - temp_dir = None - -try: - metadata = environ['META'] -except KeyError: - metadata = None - -try: - start_of_day = environ['GLBLATTR_DAY'] - start_of_day_pattern = environ['GLBLATTR_DAY_FORMAT'] -except KeyError: - start_of_day = None - start_of_day_pattern = None - -try: - time_offset = long(environ['TIME_OFFSET']) -except KeyError: - time_offset = None - -@contextmanager -def closing(thing): - try: - yield thing - finally: - thing.close() - - -def parse_input(the_input): - # Split string on ';' - specs_and_path = [str(part).strip() for part in str(the_input).split(';')] - - # Tile specifications are all but the last element - specs = specs_and_path[:-1] - # Generate a list of tuples, where each tuple is a (string, map) that represents a - # tile spec in the form (str(section_spec), { dimension_name : slice, dimension2_name : slice }) - tile_specifications = [slices_from_spec(section_spec) for section_spec in specs] - - # The path is the last element of the input split by ';' - file_path = specs_and_path[-1] - file_name = file_path.split(sep)[-1] - # If given a temporary directory location, copy the file to the temporary directory and return that path - if temp_dir is not None: - temp_file_path = path.join(temp_dir, file_name) - with closing(urllib2.urlopen(file_path)) as original_granule: - with open(temp_file_path, 'wb') as temp_granule: - for chunk in iter((lambda: original_granule.read(512000)), ''): - temp_granule.write(chunk) - - file_path = temp_file_path - - # Remove file:// if it's there because netcdf lib doesn't like it - file_path = file_path[len('file://'):] if file_path.startswith('file://') else file_path - - return tile_specifications, file_path - - -def slices_from_spec(spec): - dimtoslice = {} - for dimension in spec.split(','): - name, start, stop = dimension.split(':') - dimtoslice[name] = slice(int(start), int(stop)) - - return spec, dimtoslice - - -def to_seconds_from_epoch(date, timeunits=None, start_day=None, timeoffset=None): - try: - date = num2date(date, units=timeunits) - except ValueError: - assert isinstance(start_day, datetime.date), "start_day is not a datetime.date object" - the_datetime = datetime.datetime.combine(start_day, datetime.datetime.min.time()) - date = the_datetime + datetime.timedelta(seconds=date) - - if isinstance(date, datetime.datetime): - date = timezone('UTC').localize(date) - else: - date = timezone('UTC').localize(datetime.datetime.strptime(str(date), '%Y-%m-%d %H:%M:%S')) - - if timeoffset is not None: - return long((date - EPOCH).total_seconds()) + timeoffset - else: - return long((date - EPOCH).total_seconds()) - - -def get_ordered_slices(ds, variable, dimension_to_slice): - dimensions_for_variable = [str(dimension) for dimension in ds[variable].dimensions] - ordered_slices = OrderedDict() - for dimension in dimensions_for_variable: - ordered_slices[dimension] = dimension_to_slice[dimension] - return ordered_slices - - -def new_nexus_tile(file_path, section_spec): - nexus_tile = nexusproto.NexusTile() - tile_summary = nexusproto.TileSummary() - tile_summary.granule = file_path.split(sep)[-1] - tile_summary.section_spec = section_spec - nexus_tile.summary.CopyFrom(tile_summary) - return nexus_tile - - -def read_grid_data(self, section_spec_dataset): - tile_specifications, file_path = parse_input(section_spec_dataset) - - # Time is optional for Grid data - try: - time = environ['TIME'] - except KeyError: - time = None - - with Dataset(file_path) as ds: - for section_spec, dimtoslice in tile_specifications: - tile = nexusproto.GridTile() - - tile.latitude.CopyFrom(to_shaped_array(numpy.ma.filled(ds[latitude][dimtoslice[latitude]], numpy.NaN))) - - tile.longitude.CopyFrom(to_shaped_array(numpy.ma.filled(ds[longitude][dimtoslice[longitude]], numpy.NaN))) - - # Before we read the data we need to make sure the dimensions are in the proper order so we don't have any - # indexing issues - ordered_slices = get_ordered_slices(ds, variable_to_read, dimtoslice) - # Read data using the ordered slices, replacing masked values with NaN - data_array = numpy.ma.filled(ds[variable_to_read][tuple(ordered_slices.itervalues())], numpy.NaN) - - tile.variable_data.CopyFrom(to_shaped_array(data_array)) - - if metadata is not None: - tile.meta_data.add().CopyFrom(to_metadata(metadata, ds[metadata][tuple(ordered_slices.itervalues())])) - - if time is not None: - timevar = ds[time] - # Note assumption is that index of time is start value in dimtoslice - tile.time = to_seconds_from_epoch(timevar[dimtoslice[time].start], timeunits=timevar.getncattr('units'), timeoffset=time_offset) - - nexus_tile = new_nexus_tile(file_path, section_spec) - nexus_tile.tile.grid_tile.CopyFrom(tile) - - yield nexus_tile.SerializeToString() - - # If temp dir is defined, delete the temporary file - if temp_dir is not None: - remove(file_path) - - -def read_swath_data(self, section_spec_dataset): - tile_specifications, file_path = parse_input(section_spec_dataset) - - # Time is required for swath data - time = environ['TIME'] - - with Dataset(file_path) as ds: - for section_spec, dimtoslice in tile_specifications: - tile = nexusproto.SwathTile() - # Time Lat Long Data and metadata should all be indexed by the same dimensions, order the incoming spec once using the data variable - ordered_slices = get_ordered_slices(ds, variable_to_read, dimtoslice) - tile.latitude.CopyFrom( - to_shaped_array(numpy.ma.filled(ds[latitude][tuple(ordered_slices.itervalues())], numpy.NaN))) - - tile.longitude.CopyFrom( - to_shaped_array(numpy.ma.filled(ds[longitude][tuple(ordered_slices.itervalues())], numpy.NaN))) - - timetile = ds[time][tuple([ordered_slices[time_dim] for time_dim in ds[time].dimensions])].astype('float64', - casting='same_kind', - copy=False) - timeunits = ds[time].getncattr('units') - try: - start_of_day_date = datetime.datetime.strptime(ds.getncattr(start_of_day), start_of_day_pattern) - except Exception: - start_of_day_date = None - - for index in numpy.ndindex(timetile.shape): - timetile[index] = to_seconds_from_epoch(timetile[index].item(), timeunits=timeunits, - start_day=start_of_day_date, timeoffset=time_offset) - - tile.time.CopyFrom(to_shaped_array(timetile)) - - # Read the data converting masked values to NaN - data_array = numpy.ma.filled(ds[variable_to_read][tuple(ordered_slices.itervalues())], numpy.NaN) - tile.variable_data.CopyFrom(to_shaped_array(data_array)) - - if metadata is not None: - tile.meta_data.add().CopyFrom(to_metadata(metadata, ds[metadata][tuple(ordered_slices.itervalues())])) - - nexus_tile = new_nexus_tile(file_path, section_spec) - nexus_tile.tile.swath_tile.CopyFrom(tile) - - yield nexus_tile.SerializeToString() - - # If temp dir is defined, delete the temporary file - if temp_dir is not None: - remove(file_path) - - -def start(): - reader_types = { - 'GRIDTILE': read_grid_data, - 'SWATHTILE': read_swath_data - } - - try: - read_method = reader_types[reader_type] - except KeyError as ke: - raise NotImplementedError('Environment variable READER must be one of: [%s] but it was ''%s''' % ( - ','.join(reader_types.keys()), reader_type)) - - start_server(read_method, LengthHeaderTcpProcessor) - - -if __name__ == "__main__": - start() http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/test/java/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.java b/src/test/java/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.java index f486419..ab6735f 100644 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.java +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.java @@ -1,54 +1,16 @@ package gov.nasa.jpl.nexus.ningester; -import gov.nasa.jpl.nexus.ningester.configuration.BatchConfig; -import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.core.JobParametersBuilder; -import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; -import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; import org.springframework.test.context.junit4.SpringRunner; -import javax.sql.DataSource; - @RunWith(SpringRunner.class) @SpringBootTest public class NingesterApplicationTests { - @Configuration - @Import({BatchConfig.class}) - static class NingesterApplicationTestsConfig{ - @Bean - JobLauncherTestUtils jobLauncherTestUtils(){ - return new JobLauncherTestUtils(); - } - - } - - @Autowired JobLauncherTestUtils jobLauncherTestUtils; - - @Test - public void contextLoads() { - } - - @Test - public void testJobCompletes() throws Exception { - - JobParameters jobParameters = new JobParametersBuilder() - .addString("granule", "classpath:granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc") - .toJobParameters(); - - JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - - Assert.assertEquals("COMPLETED", jobExecution.getExitStatus().getExitCode()); - } + @Test + public void contextLoads() { + } } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/test/java/gov/nasa/jpl/nexus/ningester/processors/PythonProcessorTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/PythonProcessorTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/processors/PythonProcessorTest.java deleted file mode 100644 index e3cdcd8..0000000 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/processors/PythonProcessorTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.processors; - -import org.junit.Test; -import org.python.core.PyFunction; -import org.python.core.PyString; -import org.python.util.PythonInterpreter; -import org.springframework.beans.factory.annotation.Autowired; - -import static org.junit.Assert.assertEquals; - -public class PythonProcessorTest { - - @Test - public void testHello(){ - PythonInterpreter interpreter = new PythonInterpreter(); - PythonProcessor processor = new PythonProcessor(interpreter, "pymodule", "capitalize"); - - String expected = "HELLO JYTHON"; - String result = processor.processWithPython("hello jython"); - - assertEquals(expected, result); - } - - @Test - public void testNumpySquare(){ - PythonInterpreter interpreter = new PythonInterpreter(); - PythonProcessor processor = new PythonProcessor(interpreter, "pymodule", "square"); - - String expected = "4"; - String result = processor.processWithPython("2"); - - assertEquals(expected, result); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java new file mode 100644 index 0000000..7de2299 --- /dev/null +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java @@ -0,0 +1,54 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.testjobs; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +@TestPropertySource(properties = { "spring.config.location = classpath:testjobs/AvhrrJobTest.yml" }) +@ActiveProfiles({"test"}) +public class AvhrrJobTest { + + @TestConfiguration + static class NingesterApplicationTestsConfig { + + @Bean + JobLauncherTestUtils jobLauncherTestUtils() { + return new JobLauncherTestUtils(); + } + + } + + @Autowired + JobLauncherTestUtils jobLauncherTestUtils; + + + @Test + public void testJobCompletes() throws Exception { + + JobParameters jobParameters = new JobParametersBuilder() + .addString("granule", "classpath:granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc") + .toJobParameters(); + + JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); + + Assert.assertEquals("COMPLETED", jobExecution.getExitStatus().getExitCode()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/test/resources/application.properties ---------------------------------------------------------------------- diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index 132728a..fbcd88e 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -1 +1 @@ -spring.batch.job.enabled=false \ No newline at end of file +spring.batch.job.enabled=false http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/55308aec/src/test/resources/testjobs/AvhrrJobTest.yml ---------------------------------------------------------------------- diff --git a/src/test/resources/testjobs/AvhrrJobTest.yml b/src/test/resources/testjobs/AvhrrJobTest.yml new file mode 100644 index 0000000..c83bd20 --- /dev/null +++ b/src/test/resources/testjobs/AvhrrJobTest.yml @@ -0,0 +1,29 @@ +spring: + profiles: test + +ningester: + tile_processors: + - addTimeToSectionSpec + - pythonChainProcessor + addTimeToSectionSpec: + timeVar: time + ningesterpy: + base_url: http://127.0.0.1:5000/ + python_chain_properties: + uri_path: processorchain + processor_list: + - + name: GridReadingProcessor + config: + latitude: lat + longitude: lon + time: time + variable_to_read: analysed_sst + - + name: EmptyTileFilter + - + name: Subtract180Longitude + - + name: KelvinToCelsius + - + name: TileSummarizingProcessor \ No newline at end of file
