wired up the writer and updated tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/58de263f Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/58de263f Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/58de263f Branch: refs/heads/master Commit: 58de263f8c9da142eebb910931bb711ed0734aed Parents: a52c618 Author: Frank Greguska <[email protected]> Authored: Tue Jan 9 09:40:56 2018 -0800 Committer: Frank Greguska <[email protected]> Committed: Tue Jan 9 09:40:56 2018 -0800 ---------------------------------------------------------------------- build.gradle | 2 -- .../ningester/configuration/BatchConfig.java | 21 ++++++++++------ .../nexus/ningester/writer/CassandraStore.java | 2 +- .../jpl/nexus/ningester/writer/DataStore.java | 4 +-- .../jpl/nexus/ningester/writer/DynamoStore.java | 3 ++- .../nexus/ningester/writer/MetadataStore.java | 4 +-- .../jpl/nexus/ningester/writer/NexusWriter.java | 26 -------------------- .../jpl/nexus/ningester/writer/S3Store.java | 4 ++- .../jpl/nexus/ningester/writer/SolrStore.java | 2 +- .../nexus/ningester/testjobs/AvhrrJobTest.java | 9 ++++++- .../nexus/ningester/testjobs/SmapJobTest.java | 7 ++++++ 11 files changed, 40 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/58de263f/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index 7ceb035..9c61ac4 100644 --- a/build.gradle +++ b/build.gradle @@ -37,8 +37,6 @@ ext{ protobufUtilVersion = "3.5.1" netcdfJavaVersion = '4.6.9' guavaVersion = "23.2-jre" -// springDataCassandraVersion = '1.3.4.RELEASE' -// springDataSolrVersion = '2.0.2.RELEASE' } sourceSets{ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/58de263f/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java index 3e32377..d333dc7 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java @@ -4,6 +4,8 @@ import gov.nasa.jpl.nexus.ningester.configuration.properties.ApplicationProperti import gov.nasa.jpl.nexus.ningester.datatiler.FileSlicer; import gov.nasa.jpl.nexus.ningester.datatiler.NetCDFItemReader; import gov.nasa.jpl.nexus.ningester.processors.CompositeItemProcessor; +import gov.nasa.jpl.nexus.ningester.writer.DataStore; +import gov.nasa.jpl.nexus.ningester.writer.MetadataStore; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; @@ -12,9 +14,9 @@ import org.springframework.batch.core.configuration.annotation.JobBuilderFactory import org.springframework.batch.core.configuration.annotation.JobScope; import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; import org.springframework.batch.item.ItemProcessor; -import org.springframework.batch.item.ItemReader; import org.springframework.batch.item.ItemStreamReader; import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.support.CompositeItemWriter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; @@ -25,6 +27,8 @@ import org.springframework.context.annotation.Import; import org.springframework.core.io.Resource; import org.springframework.core.io.ResourceLoader; +import java.util.Arrays; + @Configuration @EnableBatchProcessing @Import(AppConfig.class) @@ -69,12 +73,15 @@ public class BatchConfig { @Bean @JobScope - protected ItemWriter<NexusContent.NexusTile> writer() { - return items -> { - for (NexusContent.NexusTile item : items) { - System.out.println("Got tile"); - } - }; + protected ItemWriter<NexusContent.NexusTile> writer(DataStore dataStore, MetadataStore metadataStore) { + + CompositeItemWriter<NexusContent.NexusTile> compositeItemWriter = new CompositeItemWriter<>(); + compositeItemWriter.setDelegates(Arrays.asList( + metadataStore::saveMetadata, + dataStore::saveData) + ); + + return compositeItemWriter; } @Bean http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/58de263f/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java index 22d5ad8..51af0bb 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/CassandraStore.java @@ -27,7 +27,7 @@ public class CassandraStore implements DataStore { } @Override - public void saveData(Collection<NexusContent.NexusTile> nexusTiles) { + public void saveData(List<? extends NexusContent.NexusTile> nexusTiles) { String query = "insert into ${tableName} (tile_id, tile_blob) VALUES (?, ?)"; cassandraTemplate.ingest(query, nexusTiles.stream() http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/58de263f/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStore.java index 8144193..499ac34 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DataStore.java @@ -8,9 +8,9 @@ package gov.nasa.jpl.nexus.ningester.writer; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; -import java.util.Collection; +import java.util.List; public interface DataStore { - void saveData(Collection<NexusContent.NexusTile> nexusTiles); + void saveData(List<? extends NexusContent.NexusTile> nexusTiles); } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/58de263f/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java index 4c13a14..43b041f 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/DynamoStore.java @@ -12,6 +12,7 @@ import com.amazonaws.services.dynamodbv2.document.Table; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import java.util.Collection; +import java.util.List; /** * Created by djsilvan on 6/26/17. @@ -28,7 +29,7 @@ public class DynamoStore implements DataStore { this.primaryKey = primaryKey; } - public void saveData(Collection<NexusContent.NexusTile> nexusTiles) { + public void saveData(List<? extends NexusContent.NexusTile> nexusTiles) { Table table = dynamoDB.getTable(tableName); http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/58de263f/src/main/java/gov/nasa/jpl/nexus/ningester/writer/MetadataStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/MetadataStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/MetadataStore.java index 5c59178..3baf6d5 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/MetadataStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/MetadataStore.java @@ -7,9 +7,9 @@ package gov.nasa.jpl.nexus.ningester.writer; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; -import java.util.Collection; +import java.util.List; public interface MetadataStore { - void saveMetadata(Collection<NexusContent.NexusTile> nexusTiles); + void saveMetadata(List<? extends NexusContent.NexusTile> nexusTiles); } http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/58de263f/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java deleted file mode 100644 index e2b53f8..0000000 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/NexusWriter.java +++ /dev/null @@ -1,26 +0,0 @@ -/***************************************************************************** - * Copyright (c) 2018 Jet Propulsion Laboratory, - * California Institute of Technology. All rights reserved - *****************************************************************************/ - -package gov.nasa.jpl.nexus.ningester.writer; - -import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; - -import java.util.Collection; - -public class NexusWriter { - - private MetadataStore metadataStore; - private DataStore dataStore; - - public NexusWriter(MetadataStore metadataStore, DataStore dataStore) { - this.metadataStore = metadataStore; - this.dataStore = dataStore; - } - - public void saveToNexus(Collection<NexusContent.NexusTile> nexusTiles) { - metadataStore.saveMetadata(nexusTiles); - dataStore.saveData(nexusTiles); - } -} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/58de263f/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java index a3f0e22..5998056 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/S3Store.java @@ -10,11 +10,13 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent.NexusTile; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.Collection; +import java.util.List; /** * Created by djsilvan on 6/26/17. @@ -29,7 +31,7 @@ public class S3Store implements DataStore { this.bucketName = bucketName; } - public void saveData(Collection<NexusTile> nexusTiles) { + public void saveData(List<? extends NexusTile> nexusTiles) { for (NexusTile tile : nexusTiles) { String tileId = getTileId(tile); http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/58de263f/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java index 1706b77..98a22fa 100644 --- a/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/writer/SolrStore.java @@ -45,7 +45,7 @@ public class SolrStore implements MetadataStore { } @Override - public void saveMetadata(Collection<NexusContent.NexusTile> nexusTiles) { + public void saveMetadata(List<? extends NexusContent.NexusTile> nexusTiles) { List<SolrInputDocument> solrdocs = nexusTiles.stream() .map(nexusTile -> getSolrDocFromTileSummary(nexusTile.getSummary())) http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/58de263f/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java index 7de2299..0812ffd 100644 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/AvhrrJobTest.java @@ -8,9 +8,11 @@ package gov.nasa.jpl.nexus.ningester.testjobs; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.item.ItemWriter; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -22,7 +24,7 @@ import org.springframework.test.context.junit4.SpringRunner; @RunWith(SpringRunner.class) @SpringBootTest -@TestPropertySource(properties = { "spring.config.location = classpath:testjobs/AvhrrJobTest.yml" }) +@TestPropertySource(properties = {"spring.config.location = classpath:testjobs/AvhrrJobTest.yml"}) @ActiveProfiles({"test"}) public class AvhrrJobTest { @@ -34,6 +36,11 @@ public class AvhrrJobTest { return new JobLauncherTestUtils(); } + @Bean + ItemWriter<NexusContent.NexusTile> writer() { + return items -> System.out.println("Wrote " + items.size() + " item(s)."); + } + } @Autowired http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/58de263f/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java index d99dc2b..45f94aa 100644 --- a/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/testjobs/SmapJobTest.java @@ -8,9 +8,11 @@ package gov.nasa.jpl.nexus.ningester.testjobs; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.item.ItemWriter; import org.springframework.batch.test.JobLauncherTestUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -34,6 +36,11 @@ public class SmapJobTest { return new JobLauncherTestUtils(); } + @Bean + ItemWriter<NexusContent.NexusTile> writer() { + return items -> System.out.println("Wrote " + items.size() + " item(s)."); + } + } @Autowired
