This is an automated email from the ASF dual-hosted git repository.
fgreg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ningester.git
The following commit(s) were added to refs/heads/master by this push:
new bb63acc SDAP-61 Added a test job for climatology ingestion.
bb63acc is described below
commit bb63acc8fa826b81b7f5ce553498c7f980b2fcb4
Author: fgreg <[email protected]>
AuthorDate: Tue Apr 17 15:55:45 2018 -0700
SDAP-61 Added a test job for climatology ingestion.
---
.../processors/AddDayOfYearAttributeTest.java | 4 +-
.../processors/AddTimeFromGranuleNameTest.java | 31 +++++-
.../001.L4_5day_avhrr_clim_sst_pixelMean.nc | Bin 0 -> 5221052 bytes
.../sdap/ningester/testjobs/AvhrrClimJobTest.java | 105 +++++++++++++++++++++
.../resources/testjobs/AvhrrClimJobTest.yml | 70 ++++++++++++++
5 files changed, 207 insertions(+), 3 deletions(-)
diff --git
a/src/test/java/org/apache/sdap/ningester/processors/AddDayOfYearAttributeTest.java
b/src/test/java/org/apache/sdap/ningester/processors/AddDayOfYearAttributeTest.java
index d43e0ec..a6aef61 100644
---
a/src/test/java/org/apache/sdap/ningester/processors/AddDayOfYearAttributeTest.java
+++
b/src/test/java/org/apache/sdap/ningester/processors/AddDayOfYearAttributeTest.java
@@ -36,8 +36,8 @@ public class AddDayOfYearAttributeTest {
@Test
public void testSuccessfulMatch() {
- String regex = "^(\\d{3})";
- String granuleName = "001.L4_5day_avhrr_clim_sst_pixelMean.nc";
+ String regex = "^.*(\\d{3})";
+ String granuleName =
"file:/some/path/001.L4_5day_avhrr_clim_sst_pixelMean.nc";
NexusTile nexusTile = NexusTile.newBuilder().setSummary(
TileSummary.newBuilder()
.setGranule(granuleName)
diff --git
a/src/test/java/org/apache/sdap/ningester/processors/AddTimeFromGranuleNameTest.java
b/src/test/java/org/apache/sdap/ningester/processors/AddTimeFromGranuleNameTest.java
index 4842290..86ba34d 100644
---
a/src/test/java/org/apache/sdap/ningester/processors/AddTimeFromGranuleNameTest.java
+++
b/src/test/java/org/apache/sdap/ningester/processors/AddTimeFromGranuleNameTest.java
@@ -43,7 +43,7 @@ public class AddTimeFromGranuleNameTest {
String regex = "^.*(\\d{7})\\.";
String dateFormat = "yyyyDDD";
- String granuleName = "A2012001.L3m_DAY_NSST_sst_4km.nc";
+ String granuleName =
"file:/some/path/A2012001.L3m_DAY_NSST_sst_4km.nc";
Long expectedTime = 1325376000L; // 01/01/2012 00:00:00 in epoch time
NexusTile nexusTile = NexusTile.newBuilder().setSummary(
TileSummary.newBuilder()
@@ -68,6 +68,35 @@ public class AddTimeFromGranuleNameTest {
}
@Test
+ public void testSuccessfulMatchAvhrr() {
+ String regex = "^.*(\\d{3})\\.";
+ String dateFormat = "DDD";
+
+ String granuleName =
"file:/some/path/005.L4_5day_avhrr_clim_sst_pixelMean.nc";
+ Long expectedTime = 345600L; // 01/05/1979 00:00:00 in epoch time
+ NexusTile nexusTile = NexusTile.newBuilder().setSummary(
+ TileSummary.newBuilder()
+ .setGranule(granuleName)
+ .build()
+ ).setTile(
+ TileData.newBuilder()
+ .setGridTile(
+ GridTile.newBuilder(
+
+ ).build()
+ ).build()
+ ).build();
+
+ AddTimeFromGranuleName processor = new AddTimeFromGranuleName(regex,
dateFormat);
+
+ NexusTile result = processor.setTimeFromGranuleName(nexusTile);
+
+ assertThat(result.getTile().getGridTile().getTime(), is(expectedTime));
+ assertThat(result.getSummary().getStats().getMinTime(),
is(expectedTime));
+ assertThat(result.getSummary().getStats().getMaxTime(),
is(expectedTime));
+ }
+
+ @Test
public void testUnparseable() {
String regex = "^.*(\\d{7})\\.";
String dateFormat = "yyyyDDDss";
diff --git
a/src/test/resources/granules/001.L4_5day_avhrr_clim_sst_pixelMean.nc
b/src/test/resources/granules/001.L4_5day_avhrr_clim_sst_pixelMean.nc
new file mode 100644
index 0000000..dc4e287
Binary files /dev/null and
b/src/test/resources/granules/001.L4_5day_avhrr_clim_sst_pixelMean.nc differ
diff --git
a/src/testJobs/java/org/apache/sdap/ningester/testjobs/AvhrrClimJobTest.java
b/src/testJobs/java/org/apache/sdap/ningester/testjobs/AvhrrClimJobTest.java
new file mode 100644
index 0000000..c0e52a5
--- /dev/null
+++ b/src/testJobs/java/org/apache/sdap/ningester/testjobs/AvhrrClimJobTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.sdap.ningester.testjobs;
+
+import
org.apache.sdap.ningester.configuration.properties.ApplicationProperties;
+import org.apache.sdap.ningester.configuration.properties.DatasourceProperties;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.batch.core.JobExecution;
+import org.springframework.batch.core.JobParameters;
+import org.springframework.batch.core.JobParametersBuilder;
+import org.springframework.batch.core.StepExecution;
+import org.springframework.batch.test.JobLauncherTestUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.context.annotation.Bean;
+import org.springframework.data.cassandra.core.CassandraTemplate;
+import org.springframework.data.solr.core.SolrTemplate;
+import org.springframework.data.solr.core.query.SimpleQuery;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static
org.apache.sdap.ningester.testjobs.TestUtils.assertEqualsEventually;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest
+@TestPropertySource(properties = {"spring.config.location =
classpath:testjobs/AvhrrClimJobTest.yml"})
+@ActiveProfiles({"test", "cassandra", "solr"})
+public class AvhrrClimJobTest {
+
+ @Autowired
+ JobLauncherTestUtils jobLauncherTestUtils;
+ @Autowired
+ CassandraTemplate cassandraTemplate;
+ @Autowired
+ SolrTemplate solrTemplate;
+ @Autowired
+ DatasourceProperties datasourceProperties;
+ @Autowired
+ ApplicationProperties applicationProperties;
+
+ @Before
+ public void emptyDatabase() {
+
solrTemplate.delete(datasourceProperties.getSolrStore().getCollection(), new
SimpleQuery("*:*"));
+
cassandraTemplate.truncate(datasourceProperties.getCassandraStore().getTableName());
+ }
+
+ @Test
+ public void testJobCompletes() throws Exception {
+
+ JobParameters jobParameters = new JobParametersBuilder()
+ .addString("granule",
"classpath:granules/001.L4_5day_avhrr_clim_sst_pixelMean.nc")
+ .toJobParameters();
+
+ JobExecution jobExecution =
jobLauncherTestUtils.launchJob(jobParameters);
+
+ assertThat(jobExecution.getExitStatus().getExitCode(),
is("COMPLETED"));
+ StepExecution stepExecution =
jobExecution.getStepExecutions().iterator().next();
+ assertThat(stepExecution.getReadCount(), is(1296));
+ assertThat(stepExecution.getWriteCount(), is(1053));
+ assertThat(stepExecution.getFilterCount(), is(243));
+
+ assertEqualsEventually(1053L,
+ () ->
solrTemplate.count(datasourceProperties.getSolrStore().getCollection(),
+ new SimpleQuery("dataset_s: " +
applicationProperties.getAddDatasetName().getDatasetName())),
+ 3);
+
+ long cassandraCount =
cassandraTemplate.count(datasourceProperties.getCassandraStore().getTableName());
+
+ assertThat(cassandraCount, is(1053L));
+ }
+
+ @TestConfiguration
+ static class NingesterApplicationTestsConfig {
+
+ @Bean
+ JobLauncherTestUtils jobLauncherTestUtils() {
+ return new JobLauncherTestUtils();
+ }
+
+ }
+
+
+}
diff --git a/src/testJobs/resources/testjobs/AvhrrClimJobTest.yml
b/src/testJobs/resources/testjobs/AvhrrClimJobTest.yml
new file mode 100644
index 0000000..36e52f8
--- /dev/null
+++ b/src/testJobs/resources/testjobs/AvhrrClimJobTest.yml
@@ -0,0 +1,70 @@
+# Tile Slicer Config
+ningester:
+ tile_slicer: sliceFileByTilesDesired
+ sliceFileByTilesDesired:
+ tilesDesired: 1296
+ timeDimension: time
+ dimensions:
+ - lat
+ - lon
+---
+# Tile processors configuration
+ningester:
+ tile_processors:
+ - pythonChainProcessor
+ - generateTileId
+ - addTimeFromGranuleName
+ - addDayOfYearAttribute
+ - addDatasetName
+ pythonChainProcessor:
+ enabled:
+ processor_list:
+ -
+ name: GridReadingProcessor
+ config:
+ latitude: lat
+ longitude: lon
+ time: time
+ variable_to_read: analysed_sst
+ -
+ name: EmptyTileFilter
+ -
+ name: KelvinToCelsius
+ -
+ name: TileSummarizingProcessor
+ config:
+ stored_var_name: analysed_sst
+ generateTileId:
+ enabled:
+ addTimeFromGranuleName:
+ enabled:
+ regex: ^.*(\d{3})\.
+ dateFormat: DDD
+ addDayOfYearAttribute:
+ enabled:
+ regex: ^.*(\d{3})\.
+ addDatasetName:
+ enabled:
+ datasetName: AVHRR_OI_L4_GHRSST_NCEI_CLIM
+---
+# Tile writer configuration
+ningester:
+ tile_writer:
+ data_store: cassandraStore
+ metadata_store: solrStore
+---
+# Connection settings for the test profile
+spring:
+ profiles:
+ - test
+ - local
+ data:
+ cassandra:
+ keyspaceName: nexustiles
+ contactPoints: 127.0.0.1
+ solr:
+ host: http://127.0.0.1:8983/solr/
+
+datasource:
+ solrStore:
+ collection: nexustiles
\ No newline at end of file
--
To stop receiving notification emails like this one, please contact
[email protected].
