fgreg closed pull request #1: SDAP-17: Working version of Ningester URL: https://github.com/apache/incubator-sdap-ningester/pull/1
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/.gitignore b/.gitignore index 711627e..5837aed 100644 --- a/.gitignore +++ b/.gitignore @@ -76,21 +76,6 @@ Icon Network Trash Folder Temporary Items .apdisk -### Gradle template -.gradle -/build/ - -# Ignore Gradle GUI config -gradle-app.setting - -# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) -!gradle-wrapper.jar - -# Cache of project -.gradletasknamecache - -# # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 -# gradle/wrapper/gradle-wrapper.properties ### Java template # Compiled class file *.class @@ -111,6 +96,22 @@ gradle-app.setting *.zip *.tar.gz *.rar +### Gradle template +.gradle +/build/ + +# Ignore Gradle GUI config +gradle-app.setting + +# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +!gradle-wrapper.jar + +# Cache of project +.gradletasknamecache + +# # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# gradle/wrapper/gradle-wrapper.properties + # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* diff --git a/.idea/modules/ningester_main.iml b/.idea/modules/ningester_main.iml index 39c1aa1..1a452a0 100644 --- a/.idea/modules/ningester_main.iml +++ b/.idea/modules/ningester_main.iml @@ -20,87 +20,87 @@ <orderEntry type="library" name="Gradle: org.springframework.data:spring-data-cassandra:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework.data:spring-data-solr:2.1.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: com.h2database:h2:1.4.196" level="project" /> - <orderEntry type="library" name="Gradle: org.apache.sdap:nexusproto:1.0.1-SNAPSHOT" level="project" /> + <orderEntry type="library" name="Gradle: org.apache.sdap:nexusproto:1.0.0-SNAPSHOT" level="project" /> <orderEntry type="library" name="Gradle: com.google.protobuf:protobuf-java-util:3.5.1" level="project" /> <orderEntry type="library" name="Gradle: edu.ucar:cdm:4.6.9" level="project" /> - <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-s3:1.10.77" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.data:spring-cql:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: com.google.guava:guava:19.0" level="project" /> <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-dynamodb:1.10.77" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-s3:1.10.77" level="project" /> <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter-jdbc:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework.batch:spring-batch-core:3.0.8.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-autoconfigure:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework:spring-context:4.3.13.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework:spring-aop:4.3.13.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.data:spring-data-commons:1.13.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework:spring-jdbc:4.3.13.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework:spring-tx:4.3.13.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework:spring-beans:4.3.13.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework:spring-context:4.3.13.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework:spring-core:4.3.13.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.data:spring-cql:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework:spring-expression:4.3.13.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.data:spring-data-commons:1.13.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.batch:spring-batch-infrastructure:3.0.8.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.retry:spring-retry:1.2.1.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework:spring-core:4.3.13.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: com.datastax.cassandra:cassandra-driver-core:3.1.4" level="project" /> - <orderEntry type="library" name="Gradle: org.slf4j:slf4j-api:1.7.25" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter-logging:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.slf4j:jcl-over-slf4j:1.7.25" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework:spring-tx:4.3.13.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: edu.ucar:httpservices:4.6.9" level="project" /> + <orderEntry type="library" name="Gradle: org.quartz-scheduler:quartz:2.2.0" level="project" /> + <orderEntry type="library" name="Gradle: io.dropwizard.metrics:metrics-core:3.1.5" level="project" /> + <orderEntry type="library" name="Gradle: ch.qos.logback:logback-classic:1.1.11" level="project" /> + <orderEntry type="library" name="Gradle: org.slf4j:jul-to-slf4j:1.7.25" level="project" /> + <orderEntry type="library" name="Gradle: org.slf4j:log4j-over-slf4j:1.7.25" level="project" /> + <orderEntry type="library" name="Gradle: org.slf4j:slf4j-api:1.7.25" level="project" /> <orderEntry type="library" name="Gradle: org.apache.commons:commons-lang3:3.1" level="project" /> <orderEntry type="library" name="Gradle: org.apache.solr:solr-solrj:5.5.5" level="project" /> + <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-kms:1.10.77" level="project" /> + <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-core:1.10.77" level="project" /> <orderEntry type="library" name="Gradle: com.fasterxml.jackson.core:jackson-databind:2.8.10" level="project" /> + <orderEntry type="library" name="Gradle: com.google.protobuf:protobuf-java:3.5.1" level="project" /> <orderEntry type="library" name="Gradle: com.google.code.gson:gson:2.8.2" level="project" /> <orderEntry type="library" name="Gradle: edu.ucar:udunits:4.6.9" level="project" /> - <orderEntry type="library" name="Gradle: edu.ucar:httpservices:4.6.9" level="project" /> + <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpmime:4.5.3" level="project" /> + <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpclient:4.5.3" level="project" /> <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpcore:4.4.8" level="project" /> <orderEntry type="library" name="Gradle: joda-time:joda-time:2.9.9" level="project" /> <orderEntry type="library" name="Gradle: org.jdom:jdom2:2.0.6" level="project" /> <orderEntry type="library" name="Gradle: net.jcip:jcip-annotations:1.0" level="project" /> - <orderEntry type="library" name="Gradle: org.quartz-scheduler:quartz:2.2.0" level="project" /> <orderEntry type="library" name="Gradle: com.beust:jcommander:1.35" level="project" /> - <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-kms:1.10.77" level="project" /> - <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-core:1.10.77" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot:1.5.9.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-autoconfigure:1.5.9.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter-logging:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.yaml:snakeyaml:1.17" level="project" /> <orderEntry type="library" name="Gradle: org.apache.tomcat:tomcat-jdbc:8.5.23" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework:spring-jdbc:4.3.13.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: com.ibm.jbatch:com.ibm.jbatch-tck-spi:1.0" level="project" /> <orderEntry type="library" name="Gradle: com.thoughtworks.xstream:xstream:1.4.7" level="project" /> <orderEntry type="library" name="Gradle: org.codehaus.jettison:jettison:1.2" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.batch:spring-batch-infrastructure:3.0.8.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: commons-logging:commons-logging:1.2" level="project" /> <orderEntry type="library" name="Gradle: io.netty:netty-handler:4.0.37.Final" level="project" /> - <orderEntry type="library" name="Gradle: io.dropwizard.metrics:metrics-core:3.1.5" level="project" /> - <orderEntry type="library" name="Gradle: com.github.jnr:jnr-ffi:2.0.7" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jnr-posix:3.0.27" level="project" /> + <orderEntry type="library" name="Gradle: com.github.jnr:jnr-ffi:2.0.7" level="project" /> <orderEntry type="library" name="Gradle: commons-io:commons-io:2.4" level="project" /> - <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpclient:4.5.3" level="project" /> - <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpmime:4.5.3" level="project" /> <orderEntry type="library" name="Gradle: org.apache.zookeeper:zookeeper:3.4.6" level="project" /> <orderEntry type="library" name="Gradle: org.noggit:noggit:0.6" level="project" /> <orderEntry type="library" name="Gradle: com.fasterxml.jackson.core:jackson-annotations:2.8.0" level="project" /> + <orderEntry type="library" name="Gradle: com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.8.10" level="project" /> <orderEntry type="library" name="Gradle: com.fasterxml.jackson.core:jackson-core:2.8.10" level="project" /> <orderEntry type="library" name="Gradle: c3p0:c3p0:0.9.1.1" level="project" /> - <orderEntry type="library" name="Gradle: com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.8.10" level="project" /> - <orderEntry type="library" name="Gradle: ch.qos.logback:logback-classic:1.1.11" level="project" /> - <orderEntry type="library" name="Gradle: org.slf4j:jul-to-slf4j:1.7.25" level="project" /> - <orderEntry type="library" name="Gradle: org.slf4j:log4j-over-slf4j:1.7.25" level="project" /> <orderEntry type="library" name="Gradle: org.apache.tomcat:tomcat-juli:8.5.23" level="project" /> <orderEntry type="library" name="Gradle: javax.batch:javax.batch-api:1.0" level="project" /> <orderEntry type="library" name="Gradle: xmlpull:xmlpull:1.1.3.1" level="project" /> <orderEntry type="library" name="Gradle: xpp3:xpp3_min:1.1.4c" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.retry:spring-retry:1.2.1.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: io.netty:netty-buffer:4.0.37.Final" level="project" /> - <orderEntry type="library" name="Gradle: io.netty:netty-transport:4.0.37.Final" level="project" /> <orderEntry type="library" name="Gradle: io.netty:netty-codec:4.0.37.Final" level="project" /> + <orderEntry type="library" name="Gradle: io.netty:netty-transport:4.0.37.Final" level="project" /> + <orderEntry type="library" name="Gradle: io.netty:netty-buffer:4.0.37.Final" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jffi:1.2.10" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jffi:native:1.2.10" level="project" /> - <orderEntry type="library" name="Gradle: org.ow2.asm:asm:5.0.3" level="project" /> <orderEntry type="library" name="Gradle: org.ow2.asm:asm-commons:5.0.3" level="project" /> <orderEntry type="library" name="Gradle: org.ow2.asm:asm-analysis:5.0.3" level="project" /> - <orderEntry type="library" name="Gradle: org.ow2.asm:asm-tree:5.0.3" level="project" /> <orderEntry type="library" name="Gradle: org.ow2.asm:asm-util:5.0.3" level="project" /> + <orderEntry type="library" name="Gradle: org.ow2.asm:asm-tree:5.0.3" level="project" /> + <orderEntry type="library" name="Gradle: org.ow2.asm:asm:5.0.3" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jnr-x86asm:1.0.2" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jnr-constants:0.9.0" level="project" /> <orderEntry type="library" name="Gradle: ch.qos.logback:logback-core:1.1.11" level="project" /> <orderEntry type="library" name="Gradle: io.netty:netty-common:4.0.37.Final" level="project" /> - <orderEntry type="library" name="Gradle: com.google.protobuf:protobuf-java:3.5.1" level="project" /> - <orderEntry type="library" name="Gradle: com.google.guava:guava:19.0" level="project" /> <orderEntry type="library" name="Gradle: com.vaadin.external.google:android-json:0.0.20131108.vaadin1" level="project" /> </component> <component name="PyDocumentationSettings"> diff --git a/.idea/modules/ningester_test.iml b/.idea/modules/ningester_test.iml index fc68466..4dc9233 100644 --- a/.idea/modules/ningester_test.iml +++ b/.idea/modules/ningester_test.iml @@ -22,103 +22,103 @@ <orderEntry type="library" name="Gradle: org.springframework.data:spring-data-cassandra:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework.data:spring-data-solr:2.1.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: com.h2database:h2:1.4.196" level="project" /> - <orderEntry type="library" name="Gradle: org.apache.sdap:nexusproto:1.0.1-SNAPSHOT" level="project" /> + <orderEntry type="library" name="Gradle: org.apache.sdap:nexusproto:1.0.0-SNAPSHOT" level="project" /> <orderEntry type="library" name="Gradle: com.google.protobuf:protobuf-java-util:3.5.1" level="project" /> <orderEntry type="library" name="Gradle: edu.ucar:cdm:4.6.9" level="project" /> - <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-s3:1.10.77" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.data:spring-cql:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: com.google.guava:guava:19.0" level="project" /> <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-dynamodb:1.10.77" level="project" /> + <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-s3:1.10.77" level="project" /> <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter-test:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework.batch:spring-batch-test:3.0.8.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter-jdbc:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework.batch:spring-batch-core:3.0.8.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-test-autoconfigure:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-test:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-autoconfigure:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework:spring-context:4.3.13.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework:spring-aop:4.3.13.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.data:spring-data-commons:1.13.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework:spring-jdbc:4.3.13.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework:spring-tx:4.3.13.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework:spring-beans:4.3.13.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework:spring-context:4.3.13.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework:spring-core:4.3.13.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.data:spring-cql:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework:spring-expression:4.3.13.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.data:spring-data-commons:1.13.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework:spring-test:4.3.13.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.batch:spring-batch-infrastructure:3.0.8.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.retry:spring-retry:1.2.1.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework:spring-core:4.3.13.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: com.datastax.cassandra:cassandra-driver-core:3.1.4" level="project" /> - <orderEntry type="library" name="Gradle: org.slf4j:slf4j-api:1.7.25" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter-logging:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.slf4j:jcl-over-slf4j:1.7.25" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework:spring-tx:4.3.13.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: edu.ucar:httpservices:4.6.9" level="project" /> + <orderEntry type="library" name="Gradle: org.quartz-scheduler:quartz:2.2.0" level="project" /> + <orderEntry type="library" name="Gradle: com.jayway.jsonpath:json-path:2.2.0" level="project" /> + <orderEntry type="library" name="Gradle: io.dropwizard.metrics:metrics-core:3.1.5" level="project" /> + <orderEntry type="library" name="Gradle: ch.qos.logback:logback-classic:1.1.11" level="project" /> + <orderEntry type="library" name="Gradle: org.slf4j:jul-to-slf4j:1.7.25" level="project" /> + <orderEntry type="library" name="Gradle: org.slf4j:log4j-over-slf4j:1.7.25" level="project" /> + <orderEntry type="library" name="Gradle: org.slf4j:slf4j-api:1.7.25" level="project" /> <orderEntry type="library" name="Gradle: org.apache.commons:commons-lang3:3.1" level="project" /> <orderEntry type="library" name="Gradle: org.apache.solr:solr-solrj:5.5.5" level="project" /> + <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-kms:1.10.77" level="project" /> + <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-core:1.10.77" level="project" /> <orderEntry type="library" name="Gradle: com.fasterxml.jackson.core:jackson-databind:2.8.10" level="project" /> + <orderEntry type="library" name="Gradle: com.google.protobuf:protobuf-java:3.5.1" level="project" /> <orderEntry type="library" name="Gradle: com.google.code.gson:gson:2.8.2" level="project" /> <orderEntry type="library" name="Gradle: edu.ucar:udunits:4.6.9" level="project" /> - <orderEntry type="library" name="Gradle: edu.ucar:httpservices:4.6.9" level="project" /> + <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpmime:4.5.3" level="project" /> + <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpclient:4.5.3" level="project" /> <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpcore:4.4.8" level="project" /> <orderEntry type="library" name="Gradle: joda-time:joda-time:2.9.9" level="project" /> <orderEntry type="library" name="Gradle: org.jdom:jdom2:2.0.6" level="project" /> <orderEntry type="library" name="Gradle: net.jcip:jcip-annotations:1.0" level="project" /> - <orderEntry type="library" name="Gradle: org.quartz-scheduler:quartz:2.2.0" level="project" /> <orderEntry type="library" name="Gradle: com.beust:jcommander:1.35" level="project" /> - <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-kms:1.10.77" level="project" /> - <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-core:1.10.77" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-test:1.5.9.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-test-autoconfigure:1.5.9.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: com.jayway.jsonpath:json-path:2.2.0" level="project" /> <orderEntry type="library" name="Gradle: junit:junit:4.12" level="project" /> <orderEntry type="library" name="Gradle: org.assertj:assertj-core:2.6.0" level="project" /> <orderEntry type="library" name="Gradle: org.mockito:mockito-core:1.10.19" level="project" /> - <orderEntry type="library" name="Gradle: org.hamcrest:hamcrest-core:1.3" level="project" /> <orderEntry type="library" name="Gradle: org.hamcrest:hamcrest-library:1.3" level="project" /> + <orderEntry type="library" name="Gradle: org.hamcrest:hamcrest-core:1.3" level="project" /> <orderEntry type="library" name="Gradle: org.skyscreamer:jsonassert:1.4.0" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework:spring-test:4.3.13.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: commons-collections:commons-collections:3.2.2" level="project" /> <orderEntry type="library" name="Gradle: commons-io:commons-io:2.4" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework:spring-jdbc:4.3.13.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot:1.5.9.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-autoconfigure:1.5.9.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter-logging:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.yaml:snakeyaml:1.17" level="project" /> <orderEntry type="library" name="Gradle: org.apache.tomcat:tomcat-jdbc:8.5.23" level="project" /> <orderEntry type="library" name="Gradle: com.ibm.jbatch:com.ibm.jbatch-tck-spi:1.0" level="project" /> <orderEntry type="library" name="Gradle: com.thoughtworks.xstream:xstream:1.4.7" level="project" /> <orderEntry type="library" name="Gradle: org.codehaus.jettison:jettison:1.2" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.batch:spring-batch-infrastructure:3.0.8.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: commons-logging:commons-logging:1.2" level="project" /> <orderEntry type="library" name="Gradle: io.netty:netty-handler:4.0.37.Final" level="project" /> - <orderEntry type="library" name="Gradle: io.dropwizard.metrics:metrics-core:3.1.5" level="project" /> - <orderEntry type="library" name="Gradle: com.github.jnr:jnr-ffi:2.0.7" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jnr-posix:3.0.27" level="project" /> - <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpclient:4.5.3" level="project" /> - <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpmime:4.5.3" level="project" /> + <orderEntry type="library" name="Gradle: com.github.jnr:jnr-ffi:2.0.7" level="project" /> <orderEntry type="library" name="Gradle: org.apache.zookeeper:zookeeper:3.4.6" level="project" /> <orderEntry type="library" name="Gradle: org.noggit:noggit:0.6" level="project" /> <orderEntry type="library" name="Gradle: com.fasterxml.jackson.core:jackson-annotations:2.8.0" level="project" /> + <orderEntry type="library" name="Gradle: com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.8.10" level="project" /> <orderEntry type="library" name="Gradle: com.fasterxml.jackson.core:jackson-core:2.8.10" level="project" /> <orderEntry type="library" name="Gradle: c3p0:c3p0:0.9.1.1" level="project" /> - <orderEntry type="library" name="Gradle: com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.8.10" level="project" /> <orderEntry type="library" name="Gradle: net.minidev:json-smart:2.2.1" level="project" /> <orderEntry type="library" name="Gradle: org.objenesis:objenesis:2.1" level="project" /> - <orderEntry type="library" name="Gradle: ch.qos.logback:logback-classic:1.1.11" level="project" /> - <orderEntry type="library" name="Gradle: org.slf4j:jul-to-slf4j:1.7.25" level="project" /> - <orderEntry type="library" name="Gradle: org.slf4j:log4j-over-slf4j:1.7.25" level="project" /> <orderEntry type="library" name="Gradle: org.apache.tomcat:tomcat-juli:8.5.23" level="project" /> <orderEntry type="library" name="Gradle: javax.batch:javax.batch-api:1.0" level="project" /> <orderEntry type="library" name="Gradle: xmlpull:xmlpull:1.1.3.1" level="project" /> <orderEntry type="library" name="Gradle: xpp3:xpp3_min:1.1.4c" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.retry:spring-retry:1.2.1.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: io.netty:netty-buffer:4.0.37.Final" level="project" /> - <orderEntry type="library" name="Gradle: io.netty:netty-transport:4.0.37.Final" level="project" /> <orderEntry type="library" name="Gradle: io.netty:netty-codec:4.0.37.Final" level="project" /> + <orderEntry type="library" name="Gradle: io.netty:netty-transport:4.0.37.Final" level="project" /> + <orderEntry type="library" name="Gradle: io.netty:netty-buffer:4.0.37.Final" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jffi:1.2.10" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jffi:native:1.2.10" level="project" /> - <orderEntry type="library" name="Gradle: org.ow2.asm:asm:5.0.3" level="project" /> <orderEntry type="library" name="Gradle: org.ow2.asm:asm-commons:5.0.3" level="project" /> <orderEntry type="library" name="Gradle: org.ow2.asm:asm-analysis:5.0.3" level="project" /> - <orderEntry type="library" name="Gradle: org.ow2.asm:asm-tree:5.0.3" level="project" /> <orderEntry type="library" name="Gradle: org.ow2.asm:asm-util:5.0.3" level="project" /> + <orderEntry type="library" name="Gradle: org.ow2.asm:asm-tree:5.0.3" level="project" /> + <orderEntry type="library" name="Gradle: net.minidev:accessors-smart:1.1" level="project" /> + <orderEntry type="library" name="Gradle: org.ow2.asm:asm:5.0.3" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jnr-x86asm:1.0.2" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jnr-constants:0.9.0" level="project" /> - <orderEntry type="library" name="Gradle: net.minidev:accessors-smart:1.1" level="project" /> <orderEntry type="library" name="Gradle: ch.qos.logback:logback-core:1.1.11" level="project" /> <orderEntry type="library" name="Gradle: io.netty:netty-common:4.0.37.Final" level="project" /> - <orderEntry type="library" name="Gradle: com.google.protobuf:protobuf-java:3.5.1" level="project" /> - <orderEntry type="library" name="Gradle: com.google.guava:guava:19.0" level="project" /> </component> <component name="TestModuleProperties" production-module="ningester_main" /> </module> \ No newline at end of file diff --git a/.idea/modules/ningester_testJobs.iml b/.idea/modules/ningester_testJobs.iml index 004ed70..67c8b84 100644 --- a/.idea/modules/ningester_testJobs.iml +++ b/.idea/modules/ningester_testJobs.iml @@ -22,102 +22,102 @@ <orderEntry type="library" name="Gradle: org.springframework.data:spring-data-cassandra:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework.data:spring-data-solr:2.1.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: com.h2database:h2:1.4.196" level="project" /> - <orderEntry type="library" name="Gradle: org.apache.sdap:nexusproto:1.0.1-SNAPSHOT" level="project" /> + <orderEntry type="library" name="Gradle: org.apache.sdap:nexusproto:1.0.0-SNAPSHOT" level="project" /> <orderEntry type="library" name="Gradle: com.google.protobuf:protobuf-java-util:3.5.1" level="project" /> <orderEntry type="library" name="Gradle: edu.ucar:cdm:4.6.9" level="project" /> - <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-s3:1.10.77" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.data:spring-cql:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: com.google.guava:guava:19.0" level="project" /> <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-dynamodb:1.10.77" level="project" /> + <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-s3:1.10.77" level="project" /> <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter-test:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework.batch:spring-batch-test:3.0.8.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter-jdbc:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework.batch:spring-batch-core:3.0.8.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-test-autoconfigure:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-test:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-autoconfigure:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot:1.5.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework:spring-context:4.3.13.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework:spring-aop:4.3.13.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.data:spring-data-commons:1.13.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework:spring-jdbc:4.3.13.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework:spring-tx:4.3.13.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework:spring-beans:4.3.13.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework:spring-context:4.3.13.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework:spring-core:4.3.13.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.data:spring-cql:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.springframework:spring-expression:4.3.13.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.data:spring-data-commons:1.13.9.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework:spring-test:4.3.13.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.batch:spring-batch-infrastructure:3.0.8.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.retry:spring-retry:1.2.1.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework:spring-core:4.3.13.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: com.datastax.cassandra:cassandra-driver-core:3.1.4" level="project" /> - <orderEntry type="library" name="Gradle: org.slf4j:slf4j-api:1.7.25" level="project" /> + <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter-logging:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.slf4j:jcl-over-slf4j:1.7.25" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework:spring-tx:4.3.13.RELEASE" level="project" /> + <orderEntry type="library" name="Gradle: edu.ucar:httpservices:4.6.9" level="project" /> + <orderEntry type="library" name="Gradle: org.quartz-scheduler:quartz:2.2.0" level="project" /> + <orderEntry type="library" name="Gradle: com.jayway.jsonpath:json-path:2.2.0" level="project" /> + <orderEntry type="library" name="Gradle: io.dropwizard.metrics:metrics-core:3.1.5" level="project" /> + <orderEntry type="library" name="Gradle: ch.qos.logback:logback-classic:1.1.11" level="project" /> + <orderEntry type="library" name="Gradle: org.slf4j:jul-to-slf4j:1.7.25" level="project" /> + <orderEntry type="library" name="Gradle: org.slf4j:log4j-over-slf4j:1.7.25" level="project" /> + <orderEntry type="library" name="Gradle: org.slf4j:slf4j-api:1.7.25" level="project" /> <orderEntry type="library" name="Gradle: org.apache.commons:commons-lang3:3.1" level="project" /> <orderEntry type="library" name="Gradle: org.apache.solr:solr-solrj:5.5.5" level="project" /> + <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-kms:1.10.77" level="project" /> + <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-core:1.10.77" level="project" /> <orderEntry type="library" name="Gradle: com.fasterxml.jackson.core:jackson-databind:2.8.10" level="project" /> + <orderEntry type="library" name="Gradle: com.google.protobuf:protobuf-java:3.5.1" level="project" /> <orderEntry type="library" name="Gradle: com.google.code.gson:gson:2.8.2" level="project" /> <orderEntry type="library" name="Gradle: edu.ucar:udunits:4.6.9" level="project" /> - <orderEntry type="library" name="Gradle: edu.ucar:httpservices:4.6.9" level="project" /> + <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpmime:4.5.3" level="project" /> + <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpclient:4.5.3" level="project" /> <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpcore:4.4.8" level="project" /> <orderEntry type="library" name="Gradle: joda-time:joda-time:2.9.9" level="project" /> <orderEntry type="library" name="Gradle: org.jdom:jdom2:2.0.6" level="project" /> <orderEntry type="library" name="Gradle: net.jcip:jcip-annotations:1.0" level="project" /> - <orderEntry type="library" name="Gradle: org.quartz-scheduler:quartz:2.2.0" level="project" /> <orderEntry type="library" name="Gradle: com.beust:jcommander:1.35" level="project" /> - <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-kms:1.10.77" level="project" /> - <orderEntry type="library" name="Gradle: com.amazonaws:aws-java-sdk-core:1.10.77" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-test:1.5.9.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-test-autoconfigure:1.5.9.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: com.jayway.jsonpath:json-path:2.2.0" level="project" /> <orderEntry type="library" name="Gradle: junit:junit:4.12" level="project" /> <orderEntry type="library" name="Gradle: org.assertj:assertj-core:2.6.0" level="project" /> <orderEntry type="library" name="Gradle: org.mockito:mockito-core:1.10.19" level="project" /> - <orderEntry type="library" name="Gradle: org.hamcrest:hamcrest-core:1.3" level="project" /> <orderEntry type="library" name="Gradle: org.hamcrest:hamcrest-library:1.3" level="project" /> + <orderEntry type="library" name="Gradle: org.hamcrest:hamcrest-core:1.3" level="project" /> <orderEntry type="library" name="Gradle: org.skyscreamer:jsonassert:1.4.0" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework:spring-test:4.3.13.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: commons-collections:commons-collections:3.2.2" level="project" /> <orderEntry type="library" name="Gradle: commons-io:commons-io:2.4" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework:spring-jdbc:4.3.13.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot:1.5.9.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-autoconfigure:1.5.9.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.boot:spring-boot-starter-logging:1.5.9.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: org.yaml:snakeyaml:1.17" level="project" /> <orderEntry type="library" name="Gradle: org.apache.tomcat:tomcat-jdbc:8.5.23" level="project" /> <orderEntry type="library" name="Gradle: com.ibm.jbatch:com.ibm.jbatch-tck-spi:1.0" level="project" /> <orderEntry type="library" name="Gradle: com.thoughtworks.xstream:xstream:1.4.7" level="project" /> <orderEntry type="library" name="Gradle: org.codehaus.jettison:jettison:1.2" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.batch:spring-batch-infrastructure:3.0.8.RELEASE" level="project" /> <orderEntry type="library" name="Gradle: commons-logging:commons-logging:1.2" level="project" /> <orderEntry type="library" name="Gradle: io.netty:netty-handler:4.0.37.Final" level="project" /> - <orderEntry type="library" name="Gradle: io.dropwizard.metrics:metrics-core:3.1.5" level="project" /> - <orderEntry type="library" name="Gradle: com.github.jnr:jnr-ffi:2.0.7" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jnr-posix:3.0.27" level="project" /> - <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpclient:4.5.3" level="project" /> - <orderEntry type="library" name="Gradle: org.apache.httpcomponents:httpmime:4.5.3" level="project" /> + <orderEntry type="library" name="Gradle: com.github.jnr:jnr-ffi:2.0.7" level="project" /> <orderEntry type="library" name="Gradle: org.apache.zookeeper:zookeeper:3.4.6" level="project" /> <orderEntry type="library" name="Gradle: org.noggit:noggit:0.6" level="project" /> <orderEntry type="library" name="Gradle: com.fasterxml.jackson.core:jackson-annotations:2.8.0" level="project" /> + <orderEntry type="library" name="Gradle: com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.8.10" level="project" /> <orderEntry type="library" name="Gradle: com.fasterxml.jackson.core:jackson-core:2.8.10" level="project" /> <orderEntry type="library" name="Gradle: c3p0:c3p0:0.9.1.1" level="project" /> - <orderEntry type="library" name="Gradle: com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.8.10" level="project" /> <orderEntry type="library" name="Gradle: net.minidev:json-smart:2.2.1" level="project" /> <orderEntry type="library" name="Gradle: org.objenesis:objenesis:2.1" level="project" /> - <orderEntry type="library" name="Gradle: ch.qos.logback:logback-classic:1.1.11" level="project" /> - <orderEntry type="library" name="Gradle: org.slf4j:jul-to-slf4j:1.7.25" level="project" /> - <orderEntry type="library" name="Gradle: org.slf4j:log4j-over-slf4j:1.7.25" level="project" /> <orderEntry type="library" name="Gradle: org.apache.tomcat:tomcat-juli:8.5.23" level="project" /> <orderEntry type="library" name="Gradle: javax.batch:javax.batch-api:1.0" level="project" /> <orderEntry type="library" name="Gradle: xmlpull:xmlpull:1.1.3.1" level="project" /> <orderEntry type="library" name="Gradle: xpp3:xpp3_min:1.1.4c" level="project" /> - <orderEntry type="library" name="Gradle: org.springframework.retry:spring-retry:1.2.1.RELEASE" level="project" /> - <orderEntry type="library" name="Gradle: io.netty:netty-buffer:4.0.37.Final" level="project" /> - <orderEntry type="library" name="Gradle: io.netty:netty-transport:4.0.37.Final" level="project" /> <orderEntry type="library" name="Gradle: io.netty:netty-codec:4.0.37.Final" level="project" /> + <orderEntry type="library" name="Gradle: io.netty:netty-transport:4.0.37.Final" level="project" /> + <orderEntry type="library" name="Gradle: io.netty:netty-buffer:4.0.37.Final" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jffi:1.2.10" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jffi:native:1.2.10" level="project" /> - <orderEntry type="library" name="Gradle: org.ow2.asm:asm:5.0.3" level="project" /> <orderEntry type="library" name="Gradle: org.ow2.asm:asm-commons:5.0.3" level="project" /> <orderEntry type="library" name="Gradle: org.ow2.asm:asm-analysis:5.0.3" level="project" /> - <orderEntry type="library" name="Gradle: org.ow2.asm:asm-tree:5.0.3" level="project" /> <orderEntry type="library" name="Gradle: org.ow2.asm:asm-util:5.0.3" level="project" /> + <orderEntry type="library" name="Gradle: org.ow2.asm:asm-tree:5.0.3" level="project" /> + <orderEntry type="library" name="Gradle: net.minidev:accessors-smart:1.1" level="project" /> + <orderEntry type="library" name="Gradle: org.ow2.asm:asm:5.0.3" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jnr-x86asm:1.0.2" level="project" /> <orderEntry type="library" name="Gradle: com.github.jnr:jnr-constants:0.9.0" level="project" /> - <orderEntry type="library" name="Gradle: net.minidev:accessors-smart:1.1" level="project" /> <orderEntry type="library" name="Gradle: ch.qos.logback:logback-core:1.1.11" level="project" /> <orderEntry type="library" name="Gradle: io.netty:netty-common:4.0.37.Final" level="project" /> - <orderEntry type="library" name="Gradle: com.google.protobuf:protobuf-java:3.5.1" level="project" /> - <orderEntry type="library" name="Gradle: com.google.guava:guava:19.0" level="project" /> </component> </module> \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ae9eac8 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,130 @@ +Contributing to Apache SDAP +=========================== + +Summary +------- +This document covers how to contribute to the SDAP project. SDAP uses github PRs to manage code contributions and project manages source code development through the [SDAP JIRA instance](https://issues.apache.org/jira/browse/SDAP). +These instructions assume you have a GitHub.com account, so if you don't have one you will have to create one. Your proposed code changes will be published to your own fork of the SDAP project and you will submit a Pull Request for your changes to be added. + +_Lets get started!!!_ + +Bug fixes +--------- + +It's very important that we can easily track bug fix commits, so their hashes should remain the same in all branches. +Therefore, a pull request (PR) that fixes a bug, should be sent against a release branch. +This can be either the "current release" or the "previous release", depending on which ones are maintained. +Since the goal is a stable master, bug fixes should be "merged forward" to the next branch in order: "previous release" -> "current release" -> master (in other words: old to new) + +Developing new features +----------------------- + +Development should be done in a feature branch, branched off of master. +Send a PR(steps below) to get it into master (2x LGTM applies). +PR will only be merged when master is open, will be held otherwise until master is open again. +No back porting / cherry-picking features to existing branches! + +Fork the code +------------- + +In your browser, navigate to: [https://github.com/apache?utf8=?&q=incubator-sdap&type=&language=](https://github.com/apache?utf8=?&q=incubator-sdap&type=&language=) + +Fork whichever repository you wish to contribute to by clicking on the 'Fork' button on the top right hand side. The fork will happen and you will be taken to your own fork of the repository. Copy the Git repository URL by clicking on the clipboard next to the URL on the right hand side of the page under '**HTTPS** clone URL'. You will paste this URL when doing the following `git clone` command. + +On your computer, follow these steps to setup a local repository for working on ACS: + +``` bash +$ git clone https://github.com/YOUR_ACCOUNT/incubator-sdap-*.git +$ cd incubator-sdap-* +$ git remote add upstream https://github.com/apache/incubator-sdap-*.git +$ git checkout master +$ git fetch upstream +$ git rebase upstream/master +``` +N.B. make sure that you replace ```incubator-sdap-*``` with the actual project you wish to contribute to!!! + +Making changes +-------------- + +It is important that you create a new branch to make changes on and that you do not change the `master` branch (other than to rebase in changes from `upstream/master`). In this example we will assume you will be making your changes to a branch called `SDAP-XXX`. This `SDAP-XXX` is named after the issue you have created within the [SDAP JIRA instance](https://issues.apache.org/jira/browse/SDAP). Therefore `SDAP-XXX` will be created on your local repository and will be pushed to your forked repository on GitHub. Once this branch is on your fork you will create a Pull Request for the changes to be added to the SDAP project. + +It is best practice to create a new branch each time you want to contribute to the project and only track the changes for that pull request in this branch. + +``` bash +$ git checkout -b SDAP-XXX + (make your changes) +$ git status +$ git add . +$ git commit -a -m "SDAP-XXX Descriptive title of SDAP-XXX" +``` + +> The `-b` specifies that you want to create a new branch called `SDAP-XXX`. You only specify `-b` the first time you checkout because you are creating a new branch. Once the `SDAP-XXX` branch exists, you can later switch to it with only `git checkout SDAP-XXX`. +> Note that the commit message comprises the JIRA issue number and title... this makes explicit reference between Github and JIRA for improved project management. + + +Rebase `SDAP-XXX` to include updates from `upstream/master` +------------------------------------------------------------ + +It is important that you maintain an up-to-date `master` branch in your local repository. This is done by rebasing in the code changes from `upstream/master` (the official SDAP project repository) into your local repository. You will want to do this before you start working on a feature as well as right before you submit your changes as a pull request. We recommend you do this process periodically while you work to make sure you are working off the most recent project code. + +This process will do the following: + +1. Checkout your local `master` branch +2. Synchronize your local `master` branch with the `upstream/master` so you have all the latest changes from the project +3. Rebase the latest project code into your `SDAP-XXX` branch so it is up-to-date with the upstream code + +``` bash +$ git checkout master +$ git fetch upstream +$ git rebase upstream/master +$ git checkout SDAP-XXX +$ git rebase master +``` + +> Now your `SDAP-XXX` branch is up-to-date with all the code in `upstream/master`. + + +Make a GitHub Pull Request to contribute your changes +----------------------------------------------------- + +When you are happy with your changes and you are ready to contribute them, you will create a Pull Request on GitHub to do so. This is done by pushing your local changes to your forked repository (default remote name is `origin`) and then initiating a pull request on GitHub. + +Please include JIRA id, detailed information about the bug/feature, what all tests are executed, how the reviewer can test this feature etc. Incase of UI PRs, a screenshot is preferred. + +> **IMPORTANT:** Make sure you have rebased your `SDAP-XXX` branch to include the latest code from `upstream/master` _before_ you do this. + +``` bash +$ git push origin master +$ git push origin SDAP-XXX +``` + +Now that the `SDAP-XXX` branch has been pushed to your GitHub repository, you can initiate the pull request. + +To initiate the pull request, do the following: + +1. In your browser, navigate to your forked repository: [https://github.com/YOUR_ACCOUNT?utf8=?&q=incubator-sdap&type=&language=](https://github.com/YOUR_ACCOUNT?utf8=?&q=incubator-sdap&type=&language=), make sure you actually navigate to the specific project you wish to make the PR from. +2. Click the new button called '**Compare & pull request**' that showed up just above the main area in your forked repository +3. Validate the pull request will be into the upstream `master` and will be from your `SDAP-XXX` branch +4. Enter a detailed description of the work you have done and then click '**Send pull request**' + +If you are requested to make modifications to your proposed changes, make the changes locally on your `SDAP-XXX` branch, re-push the `SDAP-XXX` branch to your fork. The existing pull request should automatically pick up the change and update accordingly. + + +Cleaning up after a successful pull request +------------------------------------------- + +Once the `SDAP-XXX` branch has been committed into the `upstream/master` branch, your local `SDAP-XXX` branch and the `origin/SDAP-XXX` branch are no longer needed. If you want to make additional changes, restart the process with a new branch. + +> **IMPORTANT:** Make sure that your changes are in `upstream/master` before you delete your `SDAP-XXX` and `origin/SDAP-XXX` branches! + +You can delete these deprecated branches with the following: + +``` bash +$ git checkout master +$ git branch -D SDAP-XXX +$ git push origin :SDAP-XXX +``` + +Release Principles +------------------ +Coming soon \ No newline at end of file diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..f1c35ad --- /dev/null +++ b/Readme.md @@ -0,0 +1,7 @@ +# Ningester + +This application is responsible for tiling input NetCDF/HDF datasets into NEXUS tiles. + +## Contributing + +See the [CONTRIBUTING](CONTRIBUTING.md) document for more details on how to contribute to this project. \ No newline at end of file diff --git a/build.gradle b/build.gradle index faa850e..445f776 100644 --- a/build.gradle +++ b/build.gradle @@ -35,7 +35,7 @@ repositories { } ext{ - nexusprotoVersion = "1.0.1-SNAPSHOT" + nexusprotoVersion = "1.0.0-SNAPSHOT" protobufUtilVersion = "3.5.1" netcdfJavaVersion = '4.6.9' guavaVersion = "23.2-jre" diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..bec8cdc --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM openjdk:8-jdk + +MAINTAINER [email protected] + +RUN apt-get update && apt-get install -y \ + bzip2 \ + git \ + python3 \ + wget && \ + addgroup ningester && \ + adduser --system --group ningester && \ + rm -rf /var/lib/apt/lists/* && \ + chgrp ningester /usr/local + +USER ningester +WORKDIR /tmp + +# ######################## +# # Anaconda # +# ######################## +RUN wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O install_anaconda.sh && \ + /bin/bash install_anaconda.sh -b -p /usr/local/anaconda2 && \ + rm install_anaconda.sh && \ + /usr/local/anaconda2/bin/conda config --add channels conda-forge +ENV PATH /usr/local/anaconda2/bin:$PATH + + +# Install nexusproto +ARG APACHE_NEXUSPROTO=https://github.com/apache/incubator-sdap-nexusproto.git +ARG APACHE_NEXUSPROTO_BRANCH=master +COPY --chown=ningester:ningester install_nexusproto.sh ./install_nexusproto.sh +RUN ./install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH + +# Install ningesterpy +ARG APACHE_NINGESTERPY=https://github.com/apache/incubator-sdap-ningesterpy.git +ARG APACHE_NINGESTERPY_BRANCH=master +COPY --chown=ningester:ningester install_ningesterpy.sh ./install_ningesterpy.sh +RUN ./install_ningesterpy.sh $APACHE_NINGESTERPY $APACHE_NINGESTERPY_BRANCH + +# Install ningester +ARG APACHE_NINGESTER=https://github.com/apache/incubator-sdap-ningester.git +ARG APACHE_NINGESTER_BRANCH=master +COPY --chown=ningester:ningester install_ningester.sh ./install_ningester.sh +RUN ./install_ningester.sh $APACHE_NINGESTER $APACHE_NINGESTER_BRANCH && \ + export NINGESTER_JAR=`find ningester/build/libs -name ningester*.jar` + +VOLUME /data +VOLUME /config + +COPY --chown=ningester:ningester entrypoint.sh ./entrypoint.sh +ENTRYPOINT ["./entrypoint.sh"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..777a939 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,68 @@ +GitHub repository: https://github.com/apache/incubator-sdap-ningester + +# How to run this image + +The basic command is: + + docker run -it --rm -v <absolute path to config directory on host>:/config/ -v <absolute path to granule on host>:/data/<granule name> sdap/ningester <profiles to activate> + +Replacing the following: + + - `<absolute path to config directory on host>` should be the absolute path on the host to the configuration for the job + - `<absolute path to granule on host>` should be the absolute path on the host to the granule intended for ingestion + - `<granule name>` should be the filename of the granule + - `<profiles to activate>` is a comma-separated list of profiles to activate + +The [ExampleJob](example_config/ExampleJob.yml) file shows an example Job configuration that would ingest an AVHRR granule. + +## Configuration + +Upon running the image, the ningester job will scan the `/config` directory for any files that end with the `.yml` extension. Specifically it uses find: + + find /config -name "*.yml" | awk -vORS=, '{ print $1 }' + +Therefore, to configure the job, mount your configuration files into `/config` using a Docker volume. Alternatively, configuration is loaded via Spring Boot's [relaxed binding rules](https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-external-config.html#boot-features-external-config-relaxed-binding). +So, you can also configure the job through environment variables where each 'level' of the yaml file gets replaced by an '_'. + +For example, given a configuration option in yaml that looks like: + + ningester: + tile_slicer: sliceFileByTilesDesired + sliceFileByTilesDesired: + tilesDesired: 1296 + timeDimension: time + dimensions: + - lat + - lon + +These could be replaced with the following corresponding Environment variables: + + NINGESTER_TILE_SLICER=sliceFileByTilesDesired + NINGESTER_SLICE_FILE_BY_TILES_DESIRED_TILES_DESIRED=1296 + NINGESTER_SLICE_FILE_BY_TILES_DESIRED_TIME_DIMENSION=time + NINGESTER_SLICE_FILE_BY_TILES_DESIRED_DIMENSIONS[0]=lat + NINGESTER_SLICE_FILE_BY_TILES_DESIRED_DIMENSIONS[1]=lon + +However, because ningester has a lot of configuration options, it is recommended to use the yaml option. + +## Data + +Ningester is designed to ingest 1 granule per run. It looks for the granule to ingest in the `/data` directory of the container image. +Use a Docker volume to mount your data into `/data`. + +The image relies on this command to find the first file in `/data` and it will use that file for ingestion: + + find /data -type f -print -quit + +## Examples + +A few example commands are shared here. + +### Docker for Mac + +The [ConnectionSettings-DockerForMac](example_config/ConnectionSettings-DockerForMac.yml) file shows an example of how to configure the connection settings +when running this job under Docker for Mac with Solr and Cassandra running on your host Mac. + +Replace `<path to ningester>` with the path on your local workstation to the ningester github project. + + docker run -it --rm -v <path to ningester>/docker/example_config/:/config/ -v <path to ningester>/src/test/resources/granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc:/data/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc.nc sdap/ningester dockermachost,solr,cassandra \ No newline at end of file diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..250efc0 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +NINGESTER_JAR=`find ningester/build/libs -name ningester*.jar` +CONFIG_FILES=`find /config -name "*.yml" | awk -vORS=, '{ print $1 }'` +GRANULE=`find /data -type f -print -quit` + +python -m sdap.ningesterpy 2>&1 | sed "s/^/[ningesterpy] /" & + +java -Dspring.profiles.active=$1 -Dspring.config.location=classpath:/application.yml,${CONFIG_FILES} -jar ${NINGESTER_JAR} granule=file://${GRANULE} ${@:2} | sed "s/^/[ningester] /" \ No newline at end of file diff --git a/docker/example_config/ConnectionSettings-DockerForMac.yml b/docker/example_config/ConnectionSettings-DockerForMac.yml new file mode 100644 index 0000000..ad74e79 --- /dev/null +++ b/docker/example_config/ConnectionSettings-DockerForMac.yml @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# Connection settings for the dockermachost profile +spring: + profiles: + - dockermachost + data: + cassandra: + keyspaceName: nexustiles + contactPoints: docker.for.mac.localhost + solr: + host: http://docker.for.mac.localhost:8983/solr/ + +datasource: + solrStore: + collection: nexustiles \ No newline at end of file diff --git a/docker/example_config/ExampleJob.yml b/docker/example_config/ExampleJob.yml new file mode 100644 index 0000000..b9386f0 --- /dev/null +++ b/docker/example_config/ExampleJob.yml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Tile Slicer Config +ningester: + tile_slicer: sliceFileByTilesDesired + sliceFileByTilesDesired: + tilesDesired: 1296 + timeDimension: time + dimensions: + - lat + - lon + +--- +# Tile processors configuration +ningester: + tile_processors: + - pythonChainProcessor + - generateTileId + - addDatasetName + pythonChainProcessor: + enabled: + processor_list: + - + name: GridReadingProcessor + config: + latitude: lat + longitude: lon + time: time + variable_to_read: analysed_sst + - + name: EmptyTileFilter + - + name: KelvinToCelsius + - + name: TileSummarizingProcessor + generateTileId: + enabled: + addDatasetName: + enabled: + datasetName: AVHRR_OI_L4_GHRSST_NCEI +--- +# Tile writer configuration +ningester: + tile_writer: + data_store: cassandraStore + metadata_store: solrStore \ No newline at end of file diff --git a/docker/install_nexusproto.sh b/docker/install_nexusproto.sh new file mode 100755 index 0000000..e629d28 --- /dev/null +++ b/docker/install_nexusproto.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +APACHE_NEXUSPROTO="https://github.com/apache/incubator-sdap-nexusproto.git"; +MASTER="master" + +GIT_REPO=${1:-$APACHE_NEXUSPROTO} +GIT_BRANCH=${2:-$MASTER} + +mkdir nexusproto +pushd nexusproto +git init +git pull ${GIT_REPO} ${GIT_BRANCH} + +./gradlew pythonInstall --info + +./gradlew install --info +popd \ No newline at end of file diff --git a/docker/install_ningester.sh b/docker/install_ningester.sh new file mode 100755 index 0000000..9a5c7ab --- /dev/null +++ b/docker/install_ningester.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +APACHE_NINGESTER="https://github.com/apache/incubator-sdap-ningester.git"; +MASTER="master" + +GIT_REPO=${1:-APACHE_NINGESTER} +GIT_BRANCH=${2:-$MASTER} + +mkdir ningester +pushd ningester +git init +git pull ${GIT_REPO} ${GIT_BRANCH} + +./gradlew bootRepackage +popd \ No newline at end of file diff --git a/docker/install_ningesterpy.sh b/docker/install_ningesterpy.sh new file mode 100755 index 0000000..faa9e23 --- /dev/null +++ b/docker/install_ningesterpy.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +APACHE_NINGESTERPY="https://github.com/apache/incubator-sdap-ningesterpy.git"; +MASTER="master" + +GIT_REPO=${1:-APACHE_NINGESTERPY} +GIT_BRANCH=${2:-$MASTER} + +mkdir ningesterpy +pushd ningesterpy +git init +git pull ${GIT_REPO} ${GIT_BRANCH} + +python setup.py install +popd \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index abaa743..99340b4 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ec6a0c3..2c2bbe5 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,5 @@ -#Thu Oct 12 17:50:39 PDT 2017 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-3.5.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-4.4.1-bin.zip diff --git a/gradlew b/gradlew index 4453cce..cccdd3d 100755 --- a/gradlew +++ b/gradlew @@ -33,11 +33,11 @@ DEFAULT_JVM_OPTS="" # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" -warn ( ) { +warn () { echo "$*" } -die ( ) { +die () { echo echo "$*" echo @@ -155,7 +155,7 @@ if $cygwin ; then fi # Escape application args -save ( ) { +save () { for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done echo " " } diff --git a/src/main/java/org/apache/sdap/ningester/configuration/AppConfig.java b/src/main/java/org/apache/sdap/ningester/configuration/AppConfig.java index 5a0c98e..b7a7e7f 100644 --- a/src/main/java/org/apache/sdap/ningester/configuration/AppConfig.java +++ b/src/main/java/org/apache/sdap/ningester/configuration/AppConfig.java @@ -98,25 +98,6 @@ protected RestTemplate restTemplate(HttpMessageConverter nexusTileConverter) { return template; } - @Bean - public MetadataStore metadataStore() { - return new MetadataStore() { - @Override - public void saveMetadata(List<? extends NexusTile> nexusTiles) { - } - - @Override - public void deleteMetadata(List<? extends NexusTile> nexusTiles) { - } - }; - } - - @Bean - public DataStore dataStore() { - return nexusTiles -> { - }; - } - @Bean public NexusWriter nexusWriter(MetadataStore metadataStore, DataStore dataStore) { return new NexusWriter(metadataStore, dataStore); diff --git a/src/main/java/org/apache/sdap/ningester/configuration/DatasourceConfig.java b/src/main/java/org/apache/sdap/ningester/configuration/DatasourceConfig.java index c8818c5..58255ad 100644 --- a/src/main/java/org/apache/sdap/ningester/configuration/DatasourceConfig.java +++ b/src/main/java/org/apache/sdap/ningester/configuration/DatasourceConfig.java @@ -21,9 +21,12 @@ import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; import com.amazonaws.services.s3.AmazonS3Client; +import org.apache.sdap.nexusproto.NexusTile; import org.apache.sdap.ningester.configuration.properties.DatasourceProperties; import org.apache.sdap.ningester.writer.*; import org.apache.solr.client.solrj.SolrClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration; import org.springframework.boot.autoconfigure.data.cassandra.CassandraDataAutoConfiguration; @@ -39,10 +42,36 @@ import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; import javax.sql.DataSource; +import java.util.List; @Configuration public class DatasourceConfig { + private static final Logger log = LoggerFactory.getLogger(DatasourceConfig.class); + + @Bean + @Profile("default") + public MetadataStore metadataStore() { + log.warn("Default metadataStore configured. All tiles generated will not be saved to persistent storage! Enable a profile to configure metadata storage (e.g. activating the 'solr' profile will save metadata to Solr)"); + return new MetadataStore() { + @Override + public void saveMetadata(List<? extends NexusTile> nexusTiles) { + } + + @Override + public void deleteMetadata(List<? extends NexusTile> nexusTiles) { + } + }; + } + + @Bean + @Profile("default") + public DataStore dataStore() { + log.warn("Default dataStore configured. All tiles generated will not be saved to persistent storage! Enable a profile to configure data storage (e.g. activating the 'cassandra' profile will save metadata to Solr)"); + return nexusTiles -> { + }; + } + @Bean @Profile("embedded") public DataSource dataSource() { diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index db5b4ad..97823c7 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,3 +1,4 @@ +# By default, do not try to auto-configure Cassandra and Solr. Only try to configure if the solr and/or cassandra profiles are enabled. spring: profiles: default autoconfigure: @@ -6,4 +7,21 @@ spring: - org.springframework.boot.autoconfigure.data.cassandra.CassandraDataAutoConfiguration - org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration +--- +# This should not change in 99% of cases. The python process is currently designed to run on the same host, in the same container as the java process. +ningester: + pythonChainProcessor: + base_url: http://127.0.0.1:5000/ + uri_path: processorchain + +--- + +# Sensible defaults for Solr settings. +spring: + profiles: solr + +datasource: + solrStore: + commitWithin: 1500 + geoPrecision: 3 \ No newline at end of file diff --git a/src/testJobs/java/org/apache/sdap/ningester/testjobs/AvhrrJobTest.java b/src/testJobs/java/org/apache/sdap/ningester/testjobs/AvhrrJobTest.java index 498136f..baa3753 100644 --- a/src/testJobs/java/org/apache/sdap/ningester/testjobs/AvhrrJobTest.java +++ b/src/testJobs/java/org/apache/sdap/ningester/testjobs/AvhrrJobTest.java @@ -77,18 +77,18 @@ public void testJobCompletes() throws Exception { assertThat(jobExecution.getExitStatus().getExitCode(), is("COMPLETED")); StepExecution stepExecution = jobExecution.getStepExecutions().iterator().next(); - assertThat(stepExecution.getReadCount(), is(5184)); - assertThat(stepExecution.getWriteCount(), is(3904)); - assertThat(stepExecution.getFilterCount(), is(1280)); + assertThat(stepExecution.getReadCount(), is(1296)); + assertThat(stepExecution.getWriteCount(), is(1053)); + assertThat(stepExecution.getFilterCount(), is(243)); - assertEqualsEventually(3904L, + assertEqualsEventually(1053L, () -> solrTemplate.count(datasourceProperties.getSolrStore().getCollection(), new SimpleQuery("dataset_s: " + applicationProperties.getAddDatasetName().getDatasetName())), 3); long cassandraCount = cassandraTemplate.count(datasourceProperties.getCassandraStore().getTableName()); - assertThat(cassandraCount, is(3904L)); + assertThat(cassandraCount, is(1053L)); } @TestConfiguration diff --git a/src/testJobs/resources/testjobs/AvhrrJobTest.yml b/src/testJobs/resources/testjobs/AvhrrJobTest.yml index 5f6fea0..2f4b76d 100644 --- a/src/testJobs/resources/testjobs/AvhrrJobTest.yml +++ b/src/testJobs/resources/testjobs/AvhrrJobTest.yml @@ -2,7 +2,7 @@ ningester: tile_slicer: sliceFileByTilesDesired sliceFileByTilesDesired: - tilesDesired: 5184 + tilesDesired: 1296 timeDimension: time dimensions: - lat @@ -55,13 +55,6 @@ spring: solr: host: http://127.0.0.1:8983/solr/ -ningester: - pythonChainProcessor: - base_url: http://127.0.0.1:5000/ - uri_path: processorchain - datasource: solrStore: - commitWithin: 1500 - geoPrecision: 3 collection: nexustiles \ No newline at end of file diff --git a/src/testJobs/resources/testjobs/SmapJobTest.yml b/src/testJobs/resources/testjobs/SmapJobTest.yml index 6a72c6c..05b1c9f 100644 --- a/src/testJobs/resources/testjobs/SmapJobTest.yml +++ b/src/testJobs/resources/testjobs/SmapJobTest.yml @@ -53,15 +53,6 @@ spring: solr: host: http://127.0.0.1:8983/solr/ -ningester: - pythonChainProcessor: - base_url: http://127.0.0.1:5000/ - uri_path: processorchain - datasource: solrStore: - commitWithin: 1500 - geoPrecision: 3 - collection: nexustiles - cassandraStore: - tableName: sea_surface_temp \ No newline at end of file + collection: nexustiles \ No newline at end of file ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
