cleaned up buildscript and split tests into unit tests and test jobs
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/9a51dbe2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/9a51dbe2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/9a51dbe2 Branch: refs/heads/master Commit: 9a51dbe27c9ea2c4fe521aa355082bba630c4c3f Parents: 58de263 Author: Frank Greguska <[email protected]> Authored: Tue Jan 9 10:23:24 2018 -0800 Committer: Frank Greguska <[email protected]> Committed: Tue Jan 9 10:23:24 2018 -0800 ---------------------------------------------------------------------- build.gradle | 51 ++++------------ settings.gradle | 13 ++++ .../nexus/ningester/writer/CassandraStore.java | 3 +- .../nexus/ningester/testjobs/AvhrrJobTest.java | 61 ------------------- .../nexus/ningester/testjobs/SmapJobTest.java | 61 ------------------- src/test/resources/testjobs/AvhrrJobTest.yml | 57 ------------------ src/test/resources/testjobs/SmapJobTest.yml | 57 ------------------ .../nexus/ningester/testjobs/AvhrrJobTest.java | 62 ++++++++++++++++++++ .../nexus/ningester/testjobs/SmapJobTest.java | 61 +++++++++++++++++++ src/testJobs/resources/application.properties | 1 + .../resources/testjobs/AvhrrJobTest.yml | 57 ++++++++++++++++++ src/testJobs/resources/testjobs/SmapJobTest.yml | 57 ++++++++++++++++++ 12 files changed, 266 insertions(+), 275 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/9a51dbe2/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index 9c61ac4..a7e6c6d 100644 --- a/build.gradle +++ b/build.gradle @@ -1,25 +1,16 @@ -buildscript { - ext { - springBootVersion = '1.5.9.RELEASE' - } - repositories { - mavenCentral() - maven { url 'http://repo.spring.io/plugins-release' } - } - dependencies { - classpath("org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}") - classpath 'io.spring.gradle:propdeps-plugin:0.0.9.RELEASE' - } +/* + ***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +plugins { + id 'java' + id 'idea' + id 'org.springframework.boot' version '1.5.9.RELEASE' + id 'org.unbroken-dome.test-sets' version '1.4.2' } -apply plugin: 'java' -apply plugin: 'idea' -apply plugin: 'org.springframework.boot' -apply plugin: 'propdeps' -apply plugin: 'propdeps-maven' -apply plugin: 'propdeps-idea' -apply plugin: 'propdeps-eclipse' - group = 'gov.nasa.jpl.nexus.ningester' version = '0.0.1-SNAPSHOT' sourceCompatibility = 1.8 @@ -39,22 +30,8 @@ ext{ guavaVersion = "23.2-jre" } -sourceSets{ - main{ - java{ - resources{ - srcDir 'src/python' - exclude 'src/python/test' - } - } - } - test{ - java{ - resources{ - srcDir 'src/python/test' - } - } - } +testSets { + testJobs } dependencyManagement { @@ -64,8 +41,6 @@ dependencyManagement { } dependencies { - optional("org.springframework.boot:spring-boot-configuration-processor") - compile("org.springframework.boot:spring-boot-starter-batch") compile("org.springframework:spring-web") compile("org.springframework.data:spring-data-cassandra") http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/9a51dbe2/settings.gradle ---------------------------------------------------------------------- diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..82729b6 --- /dev/null +++ b/settings.gradle @@ -0,0 +1,13 @@ +pluginManagement { + resolutionStrategy { + eachPlugin { + if (requested.id.namespace?.startsWith('propdeps')) { + useModule('io.spring.gradle:propdeps-plugin:0.0.9.RELEASE') + } + } + } + repositories { + maven { url 'http://repo.spring.io/plugins-release' } + gradlePluginPortal() + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/9a51dbe2/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java index 51af0bb..492d2cf 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java @@ -1,4 +1,5 @@ -/***************************************************************************** +/* + ***************************************************************************** * Copyright (c) 2018 Jet Propulsion Laboratory, * California Institute of Technology. All rights reserved *****************************************************************************/ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/9a51dbe2/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java deleted file mode 100644 index 0812ffd..0000000 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.testjobs; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.core.JobParametersBuilder; -import org.springframework.batch.item.ItemWriter; -import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - -@RunWith(SpringRunner.class) -@SpringBootTest -@TestPropertySource(properties = {"spring.config.location = classpath:testjobs/AvhrrJobTest.yml"}) -@ActiveProfiles({"test"}) -public class AvhrrJobTest { - - @TestConfiguration - static class NingesterApplicationTestsConfig { - - @Bean - JobLauncherTestUtils jobLauncherTestUtils() { - return new JobLauncherTestUtils(); - } - - @Bean - ItemWriter<NexusContent.NexusTile> writer() { - return items -> System.out.println("Wrote " + items.size() + " item(s)."); - } - - } - - @Autowired - JobLauncherTestUtils jobLauncherTestUtils; - - - @Test - public void testJobCompletes() throws Exception { - - JobParameters jobParameters = new JobParametersBuilder() - .addString("granule", "classpath:granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc") - .toJobParameters(); - - JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - - Assert.assertEquals("COMPLETED", jobExecution.getExitStatus().getExitCode()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/9a51dbe2/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java deleted file mode 100644 index 45f94aa..0000000 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2017 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.testjobs; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; -import org.springframework.batch.core.JobExecution; -import org.springframework.batch.core.JobParameters; -import org.springframework.batch.core.JobParametersBuilder; -import org.springframework.batch.item.ItemWriter; -import org.springframework.batch.test.JobLauncherTestUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit4.SpringRunner; - -@RunWith(SpringRunner.class) -@SpringBootTest -@TestPropertySource(properties = { "spring.config.location = classpath:testjobs/SmapJobTest.yml" }) -@ActiveProfiles({"test"}) -public class SmapJobTest { - - @TestConfiguration - static class NingesterApplicationTestsConfig { - - @Bean - JobLauncherTestUtils jobLauncherTestUtils() { - return new JobLauncherTestUtils(); - } - - @Bean - ItemWriter<NexusContent.NexusTile> writer() { - return items -> System.out.println("Wrote " + items.size() + " item(s)."); - } - - } - - @Autowired - JobLauncherTestUtils jobLauncherTestUtils; - - - @Test - public void testJobCompletes() throws Exception { - - JobParameters jobParameters = new JobParametersBuilder() - .addString("granule", "classpath:granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5") - .toJobParameters(); - - JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); - - Assert.assertEquals("COMPLETED", jobExecution.getExitStatus().getExitCode()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/9a51dbe2/src/test/resources/testjobs/AvhrrJobTest.yml ---------------------------------------------------------------------- diff --git a/src/test/resources/testjobs/AvhrrJobTest.yml b/src/test/resources/testjobs/AvhrrJobTest.yml deleted file mode 100644 index 05ed43d..0000000 --- a/src/test/resources/testjobs/AvhrrJobTest.yml +++ /dev/null @@ -1,57 +0,0 @@ -spring: - profiles: test - ---- -# Tile Slicer Config -ningester: - tile_slicer: sliceFileByTilesDesired - sliceFileByTilesDesired: - tilesDesired: 5184 - timeDimension: time - dimensions: - - lat - - lon - ---- -# Tile processors configuration -ningester: - tile_processors: - - pythonChainProcessor - - generateTileId - - addDatasetName - pythonChainProcessor: - enabled: - base_url: http://127.0.0.1:5000/ - uri_path: processorchain - processor_list: - - - name: GridReadingProcessor - config: - latitude: lat - longitude: lon - time: time - variable_to_read: analysed_sst - - - name: EmptyTileFilter - - - name: KelvinToCelsius - - - name: TileSummarizingProcessor - generateTileId: - enabled: - addDatasetName: - enabled: - datasetName: AVHRR_OI_L4_GHRSST_NCEI ---- -# Tile writer configuration -ningester: - tile_writer: - data_store: cassandraStore - metadata_store: solrStore - cassandraStore: - cassandraContactPoints: 127.0.0.1 - cassandraKeyspace: nexustiles - cassandraPort: 9042 - solrStore: - solrUrl: http://127.0.0.1:8983 - solrCollection: nexustiles \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/9a51dbe2/src/test/resources/testjobs/SmapJobTest.yml ---------------------------------------------------------------------- diff --git a/src/test/resources/testjobs/SmapJobTest.yml b/src/test/resources/testjobs/SmapJobTest.yml deleted file mode 100644 index e8fda0d..0000000 --- a/src/test/resources/testjobs/SmapJobTest.yml +++ /dev/null @@ -1,57 +0,0 @@ -spring: - profiles: test - ---- -# Tile Slicer Config -ningester: - tile_slicer: sliceFileByDimension - sliceFileByDimension: - sliceByDimension: 1 - dimensionNamePrefix: phony_dim_ - dimensions: - - 0 - - 1 - ---- -# Tile processors configuration -ningester: - tile_processors: - - pythonChainProcessor - - generateTileId - - addDatasetName - pythonChainProcessor: - enabled: - base_url: http://127.0.0.1:5000/ - uri_path: processorchain - processor_list: - - - name: SwathReadingProcessor - config: - latitude: lat - longitude: lon - time: row_time - variable_to_read: smap_sss - glblattr_day: REV_START_TIME - glblattr_day_format: '%Y-%jT%H:%M:%S.%f' - - - name: EmptyTileFilter - - - name: TileSummarizingProcessor - generateTileId: - enabled: - addDatasetName: - enabled: - datasetName: SMAP_L2B_SSS ---- -# Tile writer configuration -ningester: - tile_writer: - data_store: cassandraStore - metadata_store: solrStore - cassandraStore: - cassandraContactPoints: 127.0.0.1 - cassandraKeyspace: nexustiles - cassandraPort: 9042 - solrStore: - solrUrl: http://127.0.0.1:8983 - solrCollection: nexustiles \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/9a51dbe2/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java ---------------------------------------------------------------------- diff --git a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java new file mode 100644 index 0000000..397a62d --- /dev/null +++ b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java @@ -0,0 +1,62 @@ +/* + ***************************************************************************** + * Copyright (c) 2018 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.testjobs; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +@TestPropertySource(properties = {"spring.config.location = classpath:testjobs/AvhrrJobTest.yml"}) +@ActiveProfiles({"test"}) +public class AvhrrJobTest { + + @TestConfiguration + static class NingesterApplicationTestsConfig { + + @Bean + JobLauncherTestUtils jobLauncherTestUtils() { + return new JobLauncherTestUtils(); + } + + @Bean + ItemWriter<NexusContent.NexusTile> writer() { + return items -> System.out.println("Wrote " + items.size() + " item(s)."); + } + + } + + @Autowired + JobLauncherTestUtils jobLauncherTestUtils; + + + @Test + public void testJobCompletes() throws Exception { + + JobParameters jobParameters = new JobParametersBuilder() + .addString("granule", "classpath:granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc") + .toJobParameters(); + + JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); + + Assert.assertEquals("COMPLETED", jobExecution.getExitStatus().getExitCode()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/9a51dbe2/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java ---------------------------------------------------------------------- diff --git a/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java new file mode 100644 index 0000000..45f94aa --- /dev/null +++ b/src/testJobs/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java @@ -0,0 +1,61 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.testjobs; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.test.JobLauncherTestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +@TestPropertySource(properties = { "spring.config.location = classpath:testjobs/SmapJobTest.yml" }) +@ActiveProfiles({"test"}) +public class SmapJobTest { + + @TestConfiguration + static class NingesterApplicationTestsConfig { + + @Bean + JobLauncherTestUtils jobLauncherTestUtils() { + return new JobLauncherTestUtils(); + } + + @Bean + ItemWriter<NexusContent.NexusTile> writer() { + return items -> System.out.println("Wrote " + items.size() + " item(s)."); + } + + } + + @Autowired + JobLauncherTestUtils jobLauncherTestUtils; + + + @Test + public void testJobCompletes() throws Exception { + + JobParameters jobParameters = new JobParametersBuilder() + .addString("granule", "classpath:granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5") + .toJobParameters(); + + JobExecution jobExecution = jobLauncherTestUtils.launchJob(jobParameters); + + Assert.assertEquals("COMPLETED", jobExecution.getExitStatus().getExitCode()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/9a51dbe2/src/testJobs/resources/application.properties ---------------------------------------------------------------------- diff --git a/src/testJobs/resources/application.properties b/src/testJobs/resources/application.properties new file mode 100644 index 0000000..fbcd88e --- /dev/null +++ b/src/testJobs/resources/application.properties @@ -0,0 +1 @@ +spring.batch.job.enabled=false http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/9a51dbe2/src/testJobs/resources/testjobs/AvhrrJobTest.yml ---------------------------------------------------------------------- diff --git a/src/testJobs/resources/testjobs/AvhrrJobTest.yml b/src/testJobs/resources/testjobs/AvhrrJobTest.yml new file mode 100644 index 0000000..05ed43d --- /dev/null +++ b/src/testJobs/resources/testjobs/AvhrrJobTest.yml @@ -0,0 +1,57 @@ +spring: + profiles: test + +--- +# Tile Slicer Config +ningester: + tile_slicer: sliceFileByTilesDesired + sliceFileByTilesDesired: + tilesDesired: 5184 + timeDimension: time + dimensions: + - lat + - lon + +--- +# Tile processors configuration +ningester: + tile_processors: + - pythonChainProcessor + - generateTileId + - addDatasetName + pythonChainProcessor: + enabled: + base_url: http://127.0.0.1:5000/ + uri_path: processorchain + processor_list: + - + name: GridReadingProcessor + config: + latitude: lat + longitude: lon + time: time + variable_to_read: analysed_sst + - + name: EmptyTileFilter + - + name: KelvinToCelsius + - + name: TileSummarizingProcessor + generateTileId: + enabled: + addDatasetName: + enabled: + datasetName: AVHRR_OI_L4_GHRSST_NCEI +--- +# Tile writer configuration +ningester: + tile_writer: + data_store: cassandraStore + metadata_store: solrStore + cassandraStore: + cassandraContactPoints: 127.0.0.1 + cassandraKeyspace: nexustiles + cassandraPort: 9042 + solrStore: + solrUrl: http://127.0.0.1:8983 + solrCollection: nexustiles \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/9a51dbe2/src/testJobs/resources/testjobs/SmapJobTest.yml ---------------------------------------------------------------------- diff --git a/src/testJobs/resources/testjobs/SmapJobTest.yml b/src/testJobs/resources/testjobs/SmapJobTest.yml new file mode 100644 index 0000000..e8fda0d --- /dev/null +++ b/src/testJobs/resources/testjobs/SmapJobTest.yml @@ -0,0 +1,57 @@ +spring: + profiles: test + +--- +# Tile Slicer Config +ningester: + tile_slicer: sliceFileByDimension + sliceFileByDimension: + sliceByDimension: 1 + dimensionNamePrefix: phony_dim_ + dimensions: + - 0 + - 1 + +--- +# Tile processors configuration +ningester: + tile_processors: + - pythonChainProcessor + - generateTileId + - addDatasetName + pythonChainProcessor: + enabled: + base_url: http://127.0.0.1:5000/ + uri_path: processorchain + processor_list: + - + name: SwathReadingProcessor + config: + latitude: lat + longitude: lon + time: row_time + variable_to_read: smap_sss + glblattr_day: REV_START_TIME + glblattr_day_format: '%Y-%jT%H:%M:%S.%f' + - + name: EmptyTileFilter + - + name: TileSummarizingProcessor + generateTileId: + enabled: + addDatasetName: + enabled: + datasetName: SMAP_L2B_SSS +--- +# Tile writer configuration +ningester: + tile_writer: + data_store: cassandraStore + metadata_store: solrStore + cassandraStore: + cassandraContactPoints: 127.0.0.1 + cassandraKeyspace: nexustiles + cassandraPort: 9042 + solrStore: + solrUrl: http://127.0.0.1:8983 + solrCollection: nexustiles \ No newline at end of file
