http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/build/reports/project/properties.txt ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/build/reports/project/properties.txt b/nexus-ingest/dataset-tiler/build/reports/project/properties.txt new file mode 100644 index 0000000..8fb9f33 --- /dev/null +++ b/nexus-ingest/dataset-tiler/build/reports/project/properties.txt @@ -0,0 +1,151 @@ + +------------------------------------------------------------ +Root project +------------------------------------------------------------ + +BootRepackage: class org.springframework.boot.gradle.repackage.RepackageTask +allprojects: [root project 'dataset-tiler'] +ant: org.gradle.api.internal.project.DefaultAntBuilder@32a7d330 +antBuilderFactory: org.gradle.api.internal.project.DefaultAntBuilderFactory@5b9dfc8a +applicationDefaultJvmArgs: [] +applicationDistribution: org.gradle.api.internal.file.copy.DefaultCopySpec_Decorated@6d5374d2 +applicationName: dataset-tiler +archivesBaseName: dataset-tiler +artifacts: org.gradle.api.internal.artifacts.dsl.DefaultArtifactHandler_Decorated@edb0b3 +asDynamicObject: org.gradle.api.internal.ExtensibleDynamicObject@c5fcdcd +assemble: task ':assemble' +assembleDist: task ':assembleDist' +baseClassLoaderScope: org.gradle.api.internal.initialization.DefaultClassLoaderScope@38921960 +bootRepackage: task ':bootRepackage' +bootRun: task ':bootRun' +buildDependents: task ':buildDependents' +buildDir: /Users/greguska/githubprojects/nexus/nexus-ingest/dataset-tiler/build +buildFile: /Users/greguska/githubprojects/nexus/nexus-ingest/dataset-tiler/build.gradle +buildNeeded: task ':buildNeeded' +buildScriptSource: org.gradle.groovy.scripts.UriScriptSource@4e111f44 +buildscript: org.gradle.api.internal.initialization.DefaultScriptHandler@2394461d +check: task ':check' +childProjects: {} +class: class org.gradle.api.internal.project.DefaultProject_Decorated +classLoaderScope: org.gradle.api.internal.initialization.DefaultClassLoaderScope@5157d607 +classes: task ':classes' +cleanEclipse: task ':cleanEclipse' +cleanEclipseClasspath: task ':cleanEclipseClasspath' +cleanEclipseJdt: task ':cleanEclipseJdt' +cleanEclipseProject: task ':cleanEclipseProject' +cleanIdea: task ':cleanIdea' +cleanIdeaModule: task ':cleanIdeaModule' +cleanIdeaProject: task ':cleanIdeaProject' +cleanIdeaWorkspace: task ':cleanIdeaWorkspace' +compileGroovy: task ':compileGroovy' +compileJava: task ':compileJava' +compileTestGroovy: task ':compileTestGroovy' +compileTestJava: task ':compileTestJava' +components: [org.gradle.api.internal.java.JavaLibrary@79268c54] +conf2ScopeMappings: org.gradle.api.publication.maven.internal.pom.DefaultConf2ScopeMappingContainer@60afa4c7 +configurationActions: org.gradle.configuration.project.DefaultProjectConfigurationActionContainer@3f8b470b +configurations: [configuration ':archives', configuration ':compile', configuration ':compileClasspath', configuration ':compileOnly', configuration ':default', configuration ':exported', configuration ':messageBus', configuration ':optional', configuration ':provided', configuration ':runtime', configuration ':testCompile', configuration ':testCompileClasspath', configuration ':testCompileOnly', configuration ':testRuntime', configuration ':versionManagement'] +configureModule: task ':configureModule' +convention: org.gradle.api.internal.plugins.DefaultConvention@3eae5a49 +defaultArtifacts: org.gradle.api.internal.plugins.DefaultArtifactPublicationSet_Decorated@51265475 +defaultTasks: [] +deferredProjectConfiguration: org.gradle.api.internal.project.DeferredProjectConfiguration@1caf3629 +dependencies: org.gradle.api.internal.artifacts.dsl.dependencies.DefaultDependencyHandler_Decorated@3bfd0dd5 +dependencyCacheDir: /Users/greguska/githubprojects/nexus/nexus-ingest/dataset-tiler/build/dependency-cache +dependencyCacheDirName: dependency-cache +dependencyReport: task ':dependencyReport' +depth: 0 +description: null +distTar: task ':distTar' +distZip: task ':distZip' +distributions: [org.gradle.api.distribution.internal.DefaultDistribution_Decorated@79f989a8] +distsDir: /Users/greguska/githubprojects/nexus/nexus-ingest/dataset-tiler/build/distributions +distsDirName: distributions +docsDir: /Users/greguska/githubprojects/nexus/nexus-ingest/dataset-tiler/build/docs +docsDirName: docs +eclipse: org.gradle.plugins.ide.eclipse.model.EclipseModel_Decorated@35c9e8cc +eclipseClasspath: task ':eclipseClasspath' +eclipseJdt: task ':eclipseJdt' +eclipseProject: task ':eclipseProject' +ext: org.gradle.api.internal.plugins.DefaultExtraPropertiesExtension@562f97fc +extensions: org.gradle.api.internal.plugins.DefaultConvention@3eae5a49 +fileOperations: org.gradle.api.internal.file.DefaultFileOperations@33d4fd6e +fileResolver: org.gradle.api.internal.file.BaseDirFileResolver@6d59675 +findMainClass: task ':findMainClass' +gradle: build 'dataset-tiler' +groovyRuntime: org.gradle.api.tasks.GroovyRuntime_Decorated@7c2091eb +groovydoc: task ':groovydoc' +group: org.nasa.jpl.nexus.ingest +htmlDependencyReport: task ':htmlDependencyReport' +idea: org.gradle.plugins.ide.idea.model.IdeaModel_Decorated@569a79e4 +ideaModule: task ':ideaModule' +ideaProject: task ':ideaProject' +ideaWorkspace: task ':ideaWorkspace' +inheritedScope: org.gradle.api.internal.ExtensibleDynamicObject$InheritedDynamicObject@42b1c1d4 +install: task ':install' +installApp: task ':installApp' +installDist: task ':installDist' +jar: task ':jar' +javadoc: task ':javadoc' +libsDir: /Users/greguska/githubprojects/nexus/nexus-ingest/dataset-tiler/build/libs +libsDirName: libs +logger: org.gradle.logging.internal.slf4j.OutputEventListenerBackedLogger@34e667fd +logging: org.gradle.logging.internal.DefaultLoggingManager@687d91c9 +mainClassName: +mavenPomDir: /Users/greguska/githubprojects/nexus/nexus-ingest/dataset-tiler/build/poms +modelRegistry: org.gradle.model.internal.registry.DefaultModelRegistry@10aaeae4 +modelSchemaStore: org.gradle.model.internal.manage.schema.extract.DefaultModelSchemaStore@426bb8e6 +module: org.gradle.api.internal.artifacts.ProjectBackedModule@198c3f60 +name: dataset-tiler +netcdfJavaVersion: 4.6.3 +parent: null +parentIdentifier: null +path: : +pluginManager: org.gradle.api.internal.plugins.DefaultPluginManager_Decorated@157af44c +plugins: [org.gradle.api.plugins.HelpTasksPlugin@7f8eb922, org.gradle.language.base.plugins.LifecycleBasePlugin@7ab94854, org.gradle.api.plugins.BasePlugin@15d9f6ac, org.gradle.api.plugins.ReportingBasePlugin@71a6c20c, org.gradle.platform.base.plugins.ComponentBasePlugin@1db7e233, org.gradle.language.base.plugins.LanguageBasePlugin@66d73f9c, org.gradle.platform.base.plugins.BinaryBasePlugin@3a140f61, org.gradle.api.plugins.JavaBasePlugin@111c6477, org.gradle.api.plugins.JavaPlugin@5fc5780e, org.gradle.api.plugins.GroovyBasePlugin@615d50a5, org.gradle.api.plugins.GroovyPlugin@52d01bf1, org.gradle.plugins.ide.idea.IdeaPlugin@c23fd87, org.gradle.api.plugins.MavenPlugin@32085631, org.gradle.api.distribution.plugins.DistributionPlugin@581babbc, org.gradle.api.plugins.ApplicationPlugin@6f6a6185, org.springframework.boot.gradle.SpringBootPlugin@21984069, org.springframework.build.gradle.propdep.PropDepsPlugin@7629f9e, org.springframework.build.gradle.propdep.PropDepsIdeaPlugin@221793a3, or g.gradle.plugins.ide.eclipse.EclipsePlugin@172789e0, org.springframework.build.gradle.propdep.PropDepsEclipsePlugin@2aae7f01, org.springframework.xd.gradle.plugin.ModulePlugin@364f0e2f, org.gradle.api.plugins.ProjectReportsPlugin@9f8a1f8] +processOperations: org.gradle.api.internal.file.DefaultFileOperations@33d4fd6e +processResources: task ':processResources' +processTestResources: task ':processTestResources' +project: root project 'dataset-tiler' +projectDir: /Users/greguska/githubprojects/nexus/nexus-ingest/dataset-tiler +projectEvaluationBroadcaster: ProjectEvaluationListener broadcast +projectEvaluator: org.gradle.configuration.project.LifecycleProjectEvaluator@59092856 +projectRegistry: org.gradle.api.internal.project.DefaultProjectRegistry@5dfe1967 +projectReport: task ':projectReport' +projectReportDir: /Users/greguska/githubprojects/nexus/nexus-ingest/dataset-tiler/build/reports/project +projectReportDirName: project +projects: [root project 'dataset-tiler'] +properties: {...} +propertyReport: task ':propertyReport' +reporting: org.gradle.api.reporting.ReportingExtension_Decorated@1ac435f6 +reportsDir: /Users/greguska/githubprojects/nexus/nexus-ingest/dataset-tiler/build/reports +repositories: [org.gradle.api.internal.artifacts.repositories.DefaultMavenArtifactRepository_Decorated@58de7a76, org.gradle.api.internal.artifacts.repositories.DefaultMavenArtifactRepository_Decorated@4805b2a3, org.gradle.api.internal.artifacts.repositories.DefaultMavenArtifactRepository_Decorated@651dafe5, org.gradle.api.internal.artifacts.repositories.DefaultMavenArtifactRepository_Decorated@5e4d4d35, org.gradle.api.internal.artifacts.repositories.DefaultMavenArtifactRepository_Decorated@756b8140, org.gradle.api.internal.artifacts.repositories.DefaultMavenArtifactRepository_Decorated@109eb5e4] +resources: org.gradle.api.internal.resources.DefaultResourceHandler@18745295 +rootDir: /Users/greguska/githubprojects/nexus/nexus-ingest/dataset-tiler +rootProject: root project 'dataset-tiler' +run: task ':run' +scriptHandlerFactory: org.gradle.api.internal.initialization.DefaultScriptHandlerFactory@2c64804b +scriptPluginFactory: org.gradle.configuration.DefaultScriptPluginFactory@52c7506 +serviceRegistryFactory: org.gradle.internal.service.scopes.ProjectScopeServices$4@60f8c1d2 +services: ProjectScopeServices +sourceCompatibility: 1.8 +sourceSets: [source set 'main', source set 'test'] +springBoot: org.springframework.boot.gradle.SpringBootPluginExtension_Decorated@5a22d7a2 +springIntegrationDslVersion: 1.1.2.RELEASE +springXdVersion: 1.3.1.RELEASE +standardOutputCapture: org.gradle.logging.internal.DefaultLoggingManager@687d91c9 +startScripts: task ':startScripts' +state: project state 'EXECUTED' +status: integration +subprojects: [] +targetCompatibility: 1.8 +taskReport: task ':taskReport' +tasks: [task ':assemble', task ':assembleDist', task ':bootRepackage', task ':bootRun', task ':buildDependents', task ':buildNeeded', task ':check', task ':classes', task ':cleanEclipse', task ':cleanEclipseClasspath', task ':cleanEclipseJdt', task ':cleanEclipseProject', task ':cleanIdea', task ':cleanIdeaModule', task ':cleanIdeaProject', task ':cleanIdeaWorkspace', task ':compileGroovy', task ':compileJava', task ':compileTestGroovy', task ':compileTestJava', task ':configureModule', task ':dependencyReport', task ':distTar', task ':distZip', task ':eclipse', task ':eclipseClasspath', task ':eclipseJdt', task ':eclipseProject', task ':findMainClass', task ':groovydoc', task ':htmlDependencyReport', task ':idea', task ':ideaModule', task ':ideaProject', task ':ideaWorkspace', task ':install', task ':installApp', task ':installDist', task ':jar', task ':javadoc', task ':processResources', task ':processTestResources', task ':projectReport', task ':propertyReport', task ':run', task ':startScripts', task ':taskReport', task ':test', task ':testClasses', task ':wrapper'] +test: task ':test' +testClasses: task ':testClasses' +testReportDir: /Users/greguska/githubprojects/nexus/nexus-ingest/dataset-tiler/build/reports/tests +testReportDirName: tests +testResultsDir: /Users/greguska/githubprojects/nexus/nexus-ingest/dataset-tiler/build/test-results +testResultsDirName: test-results +version: 1.0.0.BUILD-SNAPSHOT +wrapper: task ':wrapper'
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/build/reports/project/tasks.txt ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/build/reports/project/tasks.txt b/nexus-ingest/dataset-tiler/build/reports/project/tasks.txt new file mode 100644 index 0000000..8741826 --- /dev/null +++ b/nexus-ingest/dataset-tiler/build/reports/project/tasks.txt @@ -0,0 +1,82 @@ + +------------------------------------------------------------ +All tasks runnable from root project +------------------------------------------------------------ + +Application tasks +----------------- +bootRun - Run the project with support for auto-detecting main class and reloading static resources +installApp - Installs the project as a JVM application along with libs and OS specific scripts. +run - Runs this project as a JVM application + +Build tasks +----------- +assemble - Assembles the outputs of this project. +bootRepackage - Repackage existing JAR and WAR archives so that they can be executed from the command line using 'java -jar' +build - Assembles and tests this project. +buildDependents - Assembles and tests this project and all projects that depend on it. +buildNeeded - Assembles and tests this project and all projects it depends on. +classes - Assembles main classes. +clean - Deletes the build directory. +jar - Assembles a jar archive containing the main classes. +testClasses - Assembles test classes. + +Build Setup tasks +----------------- +init - Initializes a new Gradle build. [incubating] + +Distribution tasks +------------------ +assembleDist - Assembles the main distributions +distTar - Bundles the project as a distribution. +distZip - Bundles the project as a distribution. +installDist - Installs the project as a distribution as-is. + +Documentation tasks +------------------- +groovydoc - Generates Groovydoc API documentation for the main source code. +javadoc - Generates Javadoc API documentation for the main source code. + +Help tasks +---------- +buildEnvironment - Displays all buildscript dependencies declared in root project 'dataset-tiler'. +components - Displays the components produced by root project 'dataset-tiler'. [incubating] +dependencies - Displays all dependencies declared in root project 'dataset-tiler'. +dependencyInsight - Displays the insight into a specific dependency in root project 'dataset-tiler'. +help - Displays a help message. +model - Displays the configuration model of root project 'dataset-tiler'. [incubating] +projects - Displays the sub-projects of root project 'dataset-tiler'. +properties - Displays the properties of root project 'dataset-tiler'. +tasks - Displays the tasks runnable from root project 'dataset-tiler'. + +IDE tasks +--------- +cleanEclipse - Cleans all Eclipse files. +cleanIdea - Cleans IDEA project files (IML, IPR) +eclipse - Generates all Eclipse files. +idea - Generates IDEA project files (IML, IPR, IWS) + +Reporting tasks +--------------- +projectReport - Generates a report about your project. + +Verification tasks +------------------ +check - Runs all checks. +test - Runs the unit tests. + +Other tasks +----------- +cleanIdeaWorkspace +install - Installs the 'archives' artifacts into the local Maven repository. +wrapper + +Rules +----- +Pattern: clean<TaskName>: Cleans the output files of a task. +Pattern: build<ConfigurationName>: Assembles the artifacts of a configuration. +Pattern: upload<ConfigurationName>: Assembles and uploads the artifacts belonging to a configuration. + +To see all tasks and more detail, run gradle tasks --all + +To see more detail about a task, run gradle help --task <task> http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/gradle/wrapper/gradle-wrapper.jar ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/gradle/wrapper/gradle-wrapper.jar b/nexus-ingest/dataset-tiler/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..9411448 Binary files /dev/null and b/nexus-ingest/dataset-tiler/gradle/wrapper/gradle-wrapper.jar differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/gradle/wrapper/gradle-wrapper.properties ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/gradle/wrapper/gradle-wrapper.properties b/nexus-ingest/dataset-tiler/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..2d73657 --- /dev/null +++ b/nexus-ingest/dataset-tiler/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Wed Jan 04 10:25:39 PST 2017 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.12-all.zip http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/gradlew ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/gradlew b/nexus-ingest/dataset-tiler/gradlew new file mode 100755 index 0000000..9d82f78 --- /dev/null +++ b/nexus-ingest/dataset-tiler/gradlew @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; +esac + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +function splitJvmOpts() { + JVM_OPTS=("$@") +} +eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS +JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" + +exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/src/main/groovy/org/nasa/jpl/nexus/ingest/datatiler/FileSlicer.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/src/main/groovy/org/nasa/jpl/nexus/ingest/datatiler/FileSlicer.groovy b/nexus-ingest/dataset-tiler/src/main/groovy/org/nasa/jpl/nexus/ingest/datatiler/FileSlicer.groovy new file mode 100644 index 0000000..9e35b42 --- /dev/null +++ b/nexus-ingest/dataset-tiler/src/main/groovy/org/nasa/jpl/nexus/ingest/datatiler/FileSlicer.groovy @@ -0,0 +1,14 @@ +/***************************************************************************** +* Copyright (c) 2016 Jet Propulsion Laboratory, +* California Institute of Technology. All rights reserved +*****************************************************************************/ +package org.nasa.jpl.nexus.ingest.datatiler + +/** + * Created by greguska on 4/19/16. + */ +interface FileSlicer { + + def generateSlices(def inputfile) + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/src/main/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByDimension.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/src/main/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByDimension.groovy b/nexus-ingest/dataset-tiler/src/main/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByDimension.groovy new file mode 100644 index 0000000..5645370 --- /dev/null +++ b/nexus-ingest/dataset-tiler/src/main/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByDimension.groovy @@ -0,0 +1,101 @@ +/***************************************************************************** +* Copyright (c) 2016 Jet Propulsion Laboratory, +* California Institute of Technology. All rights reserved +*****************************************************************************/ +package org.nasa.jpl.nexus.ingest.datatiler + +import ucar.nc2.Variable +import ucar.nc2.dataset.NetcdfDataset + +/** + * Created by greguska on 2/1/16. + */ +class SliceFileByDimension implements FileSlicer{ + + private String sliceByDimension + private List<String> dimensions + private String dimensionNamePrefix + + public void setDimensions(List<String> dims) { + this.dimensions = dims + } + + public void setSliceByDimension(String sliceBy){ + this.sliceByDimension = sliceBy + } + + public void setDimensionNamePrefix(String dimensionNamePrefix){ + this.dimensionNamePrefix = dimensionNamePrefix + } + + def generateSlices(def inputfile) { + + return sliceByDimension.isInteger()?indexedDimensionSlicing(inputfile):namedDimensionSlicing(inputfile) + } + + def indexedDimensionSlicing(def inputfile){ + + NetcdfDataset ds = null + def dimensionNameToLength = [:] + try { + ds = NetcdfDataset.openDataset(inputfile.getAbsolutePath()) + + // Because this is indexed-based dimension slicing, the dimensions are assumed to be unlimited with no names (ie. ds.dimensions == []) + // Therefore, we need to find a 'representative' variable with dimensions that we can inspect and work with + // 'lat' and 'lon' are common variable names in the datasets we work with. So try to find one of those first + // Otherwise, just find the first variable that has the same number of dimensions as was given in this.dimensions + Variable var = (ds.getVariables().find {"lat".equalsIgnoreCase(it.shortName)})?:(ds.getVariables().find {"lon".equalsIgnoreCase(it.shortName)})?:(ds.getVariables().find { it.getDimensions().size() == this.dimensions.size()}) + + assert var != null, "Could not find a variable in ${inputfile.name} with ${dimensions.size()} dimension(s)." + + dimensions.forEach { dim_index -> + dimensionNameToLength[this.dimensionNamePrefix+dim_index] = var.getDimension(dim_index.toInteger()).length + } + } finally { + ds?.close() + } + + return generateTileBoundrySlices(this.dimensionNamePrefix+this.sliceByDimension, dimensionNameToLength) + + } + + def namedDimensionSlicing(def inputfile){ + NetcdfDataset ds = null + def dimensionNameToLength = [:] + try { + ds = NetcdfDataset.openDataset(inputfile.getAbsolutePath()) + + dimensionNameToLength = ds.getDimensions().findResults { dimension -> + + this.dimensions.contains(dimension.getShortName()) ? [(dimension.getShortName()): dimension.getLength()] : null + + }.collectEntries { it } + } finally { + ds?.close() + } + + return generateTileBoundrySlices(this.sliceByDimension, dimensionNameToLength) + } + + def generateTileBoundrySlices(sliceByDimension, dimensionNameToLength){ + def combos = dimensionNameToLength.collect { dimensionName, lengthOfDimension -> + dimensionName.equals(sliceByDimension)? + [dimensionName, lengthOfDimension, 1]: + [dimensionName, lengthOfDimension, lengthOfDimension] + }.collect { nameLengthSize -> + def dimname = nameLengthSize[0] + def length = nameLengthSize[1] + def step = nameLengthSize[2] + def bounds = [] + 0.step(length, step) { start -> + bounds.add("$dimname:$start:${start + step >= length ? length : start + step}") + } + bounds + }.combinations().collect { combo -> + combo.join(",") + } + + return combos + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/src/main/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByTilesDesired.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/src/main/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByTilesDesired.groovy b/nexus-ingest/dataset-tiler/src/main/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByTilesDesired.groovy new file mode 100644 index 0000000..861c685 --- /dev/null +++ b/nexus-ingest/dataset-tiler/src/main/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByTilesDesired.groovy @@ -0,0 +1,80 @@ +/***************************************************************************** +* Copyright (c) 2016 Jet Propulsion Laboratory, +* California Institute of Technology. All rights reserved +*****************************************************************************/ +package org.nasa.jpl.nexus.ingest.datatiler + +import ucar.nc2.dataset.NetcdfDataset +import ucar.nc2.dt.grid.GeoGrid +import ucar.nc2.ft.FeatureDatasetFactoryManager + +/** + * Created by greguska on 2/1/16. + */ +class SliceFileByTilesDesired implements FileSlicer{ + + private Integer tilesDesired + private List<String> dimensions + + public void setTilesDesired(Integer desired) { + this.tilesDesired = desired + } + + public void setDimensions(List<String> dims) { + this.dimensions = dims + } + + def generateSlices(def inputfile) { + + NetcdfDataset ds = null + def dimensionNameToLength = [:] + try { + ds = NetcdfDataset.openDataset(inputfile.getAbsolutePath()) + + dimensionNameToLength = ds.getDimensions().findResults { dimension -> + + this.dimensions.contains(dimension.getShortName()) ? [(dimension.getShortName()): dimension.getLength()] : null + + }.collectEntries { it } + } finally { + ds?.close() + } + + return generateChunkBoundrySlices(tilesDesired, dimensionNameToLength) + + } + + def generateChunkBoundrySlices(def chunksDesired, def dimensionNameToLength) { + def combos = dimensionNameToLength.collect { dimensionName, lengthOfDimension -> + [dimensionName, lengthOfDimension, calculateStepSize(lengthOfDimension, chunksDesired, dimensionNameToLength.size())] + }.collect { nameLengthSize -> + def dimname = nameLengthSize[0] + def length = nameLengthSize[1] + def step = nameLengthSize[2] + def bounds = [] + 0.step(length, step) { start -> + bounds.add("$dimname:$start:${start + step >= length ? length : start + step}") + } + bounds + }.combinations().collect { combo -> + combo.join(",") + } + + return combos + + } + + def calculateStepSize(def lengthOfDimension, def chunksDesired, def numberOfDimensions) { + return Math.floor(lengthOfDimension / (chunksDesired**(1.0 / numberOfDimensions))).toInteger() + } + + public static void main(String[] args) { + NetcdfDataset ds = NetcdfDataset.openDataset("/Users/greguska/data/mur/20150101090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc") + println ds.getCoordinateSystems()[0] + GeoGrid sst = FeatureDatasetFactoryManager.wrap(FeatureDatasetFactoryManager.findFeatureType(ds), ds, null, null).findGridByShortName("analysed_sst") + GeoGrid subset = sst.subset(null, null, new ucar.ma2.Range(2000, 2010), new ucar.ma2.Range(2500, 2510)) + + println subset + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/src/main/java/org/nasa/jpl/nexus/ingest/datatiler/DataTilerOptionsMetadata.java ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/src/main/java/org/nasa/jpl/nexus/ingest/datatiler/DataTilerOptionsMetadata.java b/nexus-ingest/dataset-tiler/src/main/java/org/nasa/jpl/nexus/ingest/datatiler/DataTilerOptionsMetadata.java new file mode 100644 index 0000000..73a9e52 --- /dev/null +++ b/nexus-ingest/dataset-tiler/src/main/java/org/nasa/jpl/nexus/ingest/datatiler/DataTilerOptionsMetadata.java @@ -0,0 +1,81 @@ +/***************************************************************************** +* Copyright (c) 2016 Jet Propulsion Laboratory, +* California Institute of Technology. All rights reserved +*****************************************************************************/ +package org.nasa.jpl.nexus.ingest.datatiler; + +import org.springframework.util.StringUtils; +import org.springframework.xd.module.options.spi.ModuleOption; +import org.springframework.xd.module.options.spi.ProfileNamesProvider; + +import javax.validation.constraints.AssertTrue; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.Pattern; + +/** + * Created by greguska on 3/1/16. + */ +public class DataTilerOptionsMetadata implements ProfileNamesProvider { + + public static final String PROPERTY_NAME_TILES_DESIRED = "tilesDesired"; + public static final String PROPERTY_NAME_DIMENSIONS = "dimensions"; + + public static final String PROPERTY_NAME_SLICE_BY_DIMENSION = "sliceByDimension"; + public static final String PROPERTY_NAME_SPLIT_RESULT = "splitResult"; + + private Integer tilesDesired; + private String dimensions; + + private String sliceByDimension; + + private Boolean splitResult = false; + + @ModuleOption("The approximate number of tiles to generate") + public void setTilesDesired(Integer tilesDesired) { + this.tilesDesired = tilesDesired; + } + + @ModuleOption("Comma-delimited list of dimensions to be used for the tiling operation") + public void setDimensions(String dimensions) { + this.dimensions = dimensions; + } + + @ModuleOption("Name of dimension to slice by") + public void setSliceByDimension(String sliceByDimension) { + this.sliceByDimension = sliceByDimension; + } + + @ModuleOption(value = "Should output specifications be split into individual messages", defaultValue = "false") + public void setSplitResult(Boolean splitResult){ this.splitResult = splitResult; } + + @Min(0) + @Max(Integer.MAX_VALUE) + public Integer getTilesDesired() { + return tilesDesired; + } + + @Pattern(regexp = "^[-\\w\\s]+(?:,[-\\w\\s]+)*$", flags = {Pattern.Flag.CASE_INSENSITIVE}, message = "Must be a comma-delimited string.") + public String getDimensions() { + return dimensions; + } + + public String getSliceByDimension() { + return this.sliceByDimension; + } + + public Boolean getSplitResult() { return this.splitResult; } + + @AssertTrue(message = "'tilesDesired' or 'sliceByDimension' are required and mutually exclusive") + private boolean isInvalid() { + return (this.tilesDesired == null && StringUtils.hasText(this.sliceByDimension)) + || (this.tilesDesired != null && !StringUtils.hasText(this.sliceByDimension)); + } + + @Override + public String[] profilesToActivate() { + return new String[]{"use-" + (this.getTilesDesired()==null?"dimension":"tilesdesired")}; + } + + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/src/main/java/org/nasa/jpl/nexus/ingest/datatiler/IntegrationConfiguration.java ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/src/main/java/org/nasa/jpl/nexus/ingest/datatiler/IntegrationConfiguration.java b/nexus-ingest/dataset-tiler/src/main/java/org/nasa/jpl/nexus/ingest/datatiler/IntegrationConfiguration.java new file mode 100644 index 0000000..71ebf0a --- /dev/null +++ b/nexus-ingest/dataset-tiler/src/main/java/org/nasa/jpl/nexus/ingest/datatiler/IntegrationConfiguration.java @@ -0,0 +1,112 @@ +/***************************************************************************** +* Copyright (c) 2016 Jet Propulsion Laboratory, +* California Institute of Technology. All rights reserved +*****************************************************************************/ +package org.nasa.jpl.nexus.ingest.datatiler; + + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.core.env.Environment; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.config.EnableIntegration; +import org.springframework.integration.dsl.IntegrationFlow; +import org.springframework.integration.dsl.IntegrationFlows; +import org.springframework.messaging.MessageChannel; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; + +import javax.annotation.Resource; +import java.util.Arrays; + +/** + * Created by greguska on 3/1/16. + */ +@Configuration +@EnableIntegration +public class IntegrationConfiguration { + + @Resource + private Environment environment; + + @Value("#{environment[T(org.nasa.jpl.nexus.ingest.datatiler.DataTilerOptionsMetadata).PROPERTY_NAME_SPLIT_RESULT]}") + private Boolean splitResult; + + @Bean + public MessageChannel input() { + return new DirectChannel(); + } + + @Bean + MessageChannel output() { + return new DirectChannel(); + } + + @Bean + public IntegrationFlow createTileSpecs(FileSlicer fileSlicer) { + return IntegrationFlows.from(this.input()) + .transform(inputFile -> + fileSlicer.generateSlices(inputFile)) + .<Object, Object>route(payload -> splitResult, mapping -> mapping + .channelMapping("false", "output") + .subFlowMapping("true", splitResultSubflow -> splitResultSubflow + .split() + )) + .channel(this.output()) + .get(); + } + + + @Bean + public TaskScheduler taskScheduler() { + ThreadPoolTaskScheduler tpts = new ThreadPoolTaskScheduler(); + tpts.setPoolSize(1); + return tpts; + } + + @Configuration + @Profile("use-tilesdesired") + static class SliceByTilesDesiredConfiguration { + @Resource + private Environment environment; + + @Value("#{environment[T(org.nasa.jpl.nexus.ingest.datatiler.DataTilerOptionsMetadata).PROPERTY_NAME_TILES_DESIRED]}") + private Integer tilesDesired; + + @Value("#{environment[T(org.nasa.jpl.nexus.ingest.datatiler.DataTilerOptionsMetadata).PROPERTY_NAME_DIMENSIONS]}") + private String dimensions; + + @Bean + public FileSlicer fileSlicer() { + SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); + slicer.setTilesDesired(tilesDesired); + slicer.setDimensions(Arrays.asList(dimensions.split(","))); + return slicer; + } + } + + @Configuration + @Profile("use-dimension") + static class SliceByDimensionConfiguration { + @Resource + private Environment environment; + + @Value("#{environment[T(org.nasa.jpl.nexus.ingest.datatiler.DataTilerOptionsMetadata).PROPERTY_NAME_SLICE_BY_DIMENSION]}") + private String sliceByDimension; + + @Value("#{environment[T(org.nasa.jpl.nexus.ingest.datatiler.DataTilerOptionsMetadata).PROPERTY_NAME_DIMENSIONS]}") + private String dimensions; + + @Bean + public FileSlicer fileSlicer() { + SliceFileByDimension slicer = new SliceFileByDimension(); + slicer.setSliceByDimension(sliceByDimension); + slicer.setDimensions(Arrays.asList(dimensions.split(","))); + slicer.setDimensionNamePrefix("phony_dim_"); + return slicer; + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/src/main/resources/config/spring-module.properties ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/src/main/resources/config/spring-module.properties b/nexus-ingest/dataset-tiler/src/main/resources/config/spring-module.properties new file mode 100644 index 0000000..2114f08 --- /dev/null +++ b/nexus-ingest/dataset-tiler/src/main/resources/config/spring-module.properties @@ -0,0 +1,2 @@ +options_class = org.nasa.jpl.nexus.ingest.datatiler.DataTilerOptionsMetadata +base_packages = org.nasa.jpl.nexus.ingest.datatiler \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/DatasetTilerIntegrationTest.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/DatasetTilerIntegrationTest.groovy b/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/DatasetTilerIntegrationTest.groovy new file mode 100644 index 0000000..42716ea --- /dev/null +++ b/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/DatasetTilerIntegrationTest.groovy @@ -0,0 +1,181 @@ +/***************************************************************************** +* Copyright (c) 2016 Jet Propulsion Laboratory, +* California Institute of Technology. All rights reserved +*****************************************************************************/ +package org.nasa.jpl.nexus.ingest.datatiler + +import org.junit.After +import org.junit.BeforeClass +import org.junit.Test +import org.springframework.core.io.ClassPathResource +import org.springframework.core.io.FileSystemResource +import org.springframework.xd.dirt.server.singlenode.SingleNodeApplication +import org.springframework.xd.dirt.test.SingleNodeIntegrationTestSupport +import org.springframework.xd.dirt.test.SingletonModuleRegistry +import org.springframework.xd.dirt.test.process.SingleNodeProcessingChain +import org.springframework.xd.module.ModuleType +import org.springframework.xd.test.RandomConfigurationSupport + +import static org.junit.Assert.assertEquals +import static org.junit.Assert.assertTrue +import static org.springframework.xd.dirt.test.process.SingleNodeProcessingChainSupport.chain; + +/** + * Created by greguska on 3/1/16. + */ +class DatasetTilerIntegrationTest { + + private static SingleNodeApplication application; + + private static int RECEIVE_TIMEOUT = 5000; + + private static String moduleName = "dataTiler"; + + SingleNodeProcessingChain chain; + + /** + * Start the single node container, binding random unused ports, etc. to not conflict with any other instances + * running on this host. Configure the ModuleRegistry to include the project module. + */ + @BeforeClass + public static void setUp() { + RandomConfigurationSupport randomConfigSupport = new RandomConfigurationSupport(); + application = new SingleNodeApplication().run(); + SingleNodeIntegrationTestSupport singleNodeIntegrationTestSupport = new SingleNodeIntegrationTestSupport + (application); + singleNodeIntegrationTestSupport.addModuleRegistry(new SingletonModuleRegistry(ModuleType.processor, + moduleName)); + + } + + @Test + public void testSliceByDesiredTiles() { + def tilesDesired = 1 + def dimensions = "lat,lon" +// def splitResult = true + + def payload = new ClassPathResource("datasets/20150101090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.nc4").getFile() + + def streamName = "testDefault" + + + def processingChainUnderTest = "$moduleName --tilesDesired=$tilesDesired --dimensions=$dimensions" + + chain = chain(application, streamName, processingChainUnderTest) + + chain.sendPayload(payload); + Object result = chain.receivePayload(RECEIVE_TIMEOUT) + + assertTrue(result instanceof List) + assertEquals(1, result.size) + assertEquals("lat:0:249,lon:0:500", result[0].toString()) + } + + @Test + public void testSliceByDimension() { + def dimensions = "NUMROWS,NUMCELLS" + def sliceByDimension = "NUMROWS" + + def payload = new ClassPathResource("datasets/ascat_20130325_002100_metopb_02676_eps_o_coa_2101_ovw.l2.nc4").getFile() + + def streamName = "testDefault" + + + def processingChainUnderTest = "$moduleName --sliceByDimension=$sliceByDimension --dimensions=$dimensions" + + chain = chain(application, streamName, processingChainUnderTest) + + chain.sendPayload(payload); + Object result = chain.receivePayload(RECEIVE_TIMEOUT) + + assertTrue(result instanceof List) + assertEquals(2, result.size) + + assertEquals("NUMROWS:0:1,NUMCELLS:0:82", result[0].toString()) + assertEquals("NUMROWS:1:2,NUMCELLS:0:82", result[1].toString()) + } + + @Test + public void testSliceByDimensionSplitResult() { + def dimensions = "NUMROWS,NUMCELLS" + def sliceByDimension = "NUMROWS" + def splitResult = true + + def payload = new ClassPathResource("datasets/ascat_20130325_002100_metopb_02676_eps_o_coa_2101_ovw.l2.nc4").getFile() + + def streamName = "testDefault" + + + def processingChainUnderTest = "$moduleName --sliceByDimension=$sliceByDimension --dimensions=$dimensions --splitResult=$splitResult" + + chain = chain(application, streamName, processingChainUnderTest) + + chain.sendPayload(payload); + Object result = chain.receivePayload(RECEIVE_TIMEOUT) + + assertTrue(result instanceof String) + assertEquals("NUMROWS:0:1,NUMCELLS:0:82", result.toString()) + + result = chain.receivePayload(RECEIVE_TIMEOUT) + + assertTrue(result instanceof String) + assertEquals("NUMROWS:1:2,NUMCELLS:0:82", result.toString()) + } + + @Test + public void testSliceByDimensionIndexedDimensions() { + def dimensions = "0,1" + def sliceByDimension = "1" + + def payload = new ClassPathResource("datasets/SMAP_L2B_SSS_00865_20150331T163144_R13080.split.h5").getFile() + + def streamName = "testDefault" + + + def processingChainUnderTest = "$moduleName --sliceByDimension=$sliceByDimension --dimensions=$dimensions" + + chain = chain(application, streamName, processingChainUnderTest) + + chain.sendPayload(payload); + Object result = chain.receivePayload(RECEIVE_TIMEOUT) + + assertTrue(result instanceof List) + assertEquals(2, result.size) + + assertEquals("phony_dim_0:0:76,phony_dim_1:0:1", result[0].toString()) + assertEquals("phony_dim_0:0:76,phony_dim_1:1:2", result[1].toString()) + } + + @Test + public void testSliceccmpByDesiredTiles() { + def tilesDesired = 1 + def dimensions = "latitude,longitude" + def splitResult = true + + def payload = new ClassPathResource("datasets/CCMP_Wind_Analysis_20160101_V02.0_L3.0_RSS.split.nc").getFile() + + def streamName = "testDefault" + + + def processingChainUnderTest = "$moduleName --tilesDesired=$tilesDesired --dimensions=$dimensions --splitResult=$splitResult" + + chain = chain(application, streamName, processingChainUnderTest) + + chain.sendPayload(payload); + Object result = chain.receivePayload(RECEIVE_TIMEOUT) + + assertTrue(result instanceof String) + assertEquals("longitude:0:87,latitude:0:38", result.toString()) + } + + /** + * Destroy the chain to reset message bus bindings and destroy the stream. + */ + @After + public void tearDown() { + if (chain != null) { + chain.destroy(); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/DatasetTilerPropertiesTest.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/DatasetTilerPropertiesTest.groovy b/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/DatasetTilerPropertiesTest.groovy new file mode 100644 index 0000000..4299906 --- /dev/null +++ b/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/DatasetTilerPropertiesTest.groovy @@ -0,0 +1,43 @@ +/***************************************************************************** +* Copyright (c) 2016 Jet Propulsion Laboratory, +* California Institute of Technology. All rights reserved +*****************************************************************************/ +package org.nasa.jpl.nexus.ingest.datatiler + +import org.hamcrest.Matcher +import org.junit.Test +import org.springframework.xd.module.ModuleDefinition +import org.springframework.xd.module.ModuleDefinitions +import org.springframework.xd.module.options.DefaultModuleOptionsMetadataResolver +import org.springframework.xd.module.options.ModuleOption +import org.springframework.xd.module.options.ModuleOptionsMetadata +import org.springframework.xd.module.options.ModuleOptionsMetadataResolver + +import static org.hamcrest.Matchers.* +import static org.junit.Assert.assertThat +import static org.springframework.xd.module.ModuleType.processor + +/** + * Created by greguska on 3/1/16. + */ +class DatasetTilerPropertiesTest { + + private String moduleName = "datasetTiler"; + + @Test + public void testModuleProperties() { + ModuleOptionsMetadataResolver moduleOptionsMetadataResolver = new DefaultModuleOptionsMetadataResolver(); + String resource = "classpath:/"; + ModuleDefinition definition = ModuleDefinitions.simple(moduleName, processor, resource); + ModuleOptionsMetadata metadata = moduleOptionsMetadataResolver.resolve(definition); + + assertThat( + metadata, + containsInAnyOrder(moduleOptionNamed("tilesDesired"), moduleOptionNamed("dimensions"), moduleOptionNamed("sliceByDimension"), moduleOptionNamed("splitResult"))); + } + + public static Matcher<ModuleOption> moduleOptionNamed(String name) { + return hasProperty("name", equalTo(name)); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByDimensionTest.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByDimensionTest.groovy b/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByDimensionTest.groovy new file mode 100644 index 0000000..eeeb9fb --- /dev/null +++ b/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByDimensionTest.groovy @@ -0,0 +1,49 @@ +/***************************************************************************** +* Copyright (c) 2016 Jet Propulsion Laboratory, +* California Institute of Technology. All rights reserved +*****************************************************************************/ +package org.nasa.jpl.nexus.ingest.datatiler + +import org.junit.Test + +import static junit.framework.Assert.assertEquals +import static org.hamcrest.CoreMatchers.hasItems +import static org.hamcrest.MatcherAssert.assertThat + +/** + * Created by greguska on 4/18/16. + */ +class SliceFileByDimensionTest { + + @Test + public void testGenerateTileBoundrySlices(){ + + def slicer = new SliceFileByDimension() + slicer.setSliceByDimension("NUMROWS") + + def dimensionNameToLength = ['NUMROWS':3163, 'NUMCELLS':82] + + def result = slicer.generateTileBoundrySlices("NUMROWS", dimensionNameToLength) + + assertEquals(3163, result.size) + + assertThat(result, hasItems("NUMROWS:0:1,NUMCELLS:0:82", "NUMROWS:1:2,NUMCELLS:0:82", "NUMROWS:3162:3163,NUMCELLS:0:82")) + + } + + @Test + public void testGenerateTileBoundrySlices2(){ + + def slicer = new SliceFileByDimension() + slicer.setSliceByDimension("NUMROWS") + + def dimensionNameToLength = ['NUMROWS':2, 'NUMCELLS':82] + + def result = slicer.generateTileBoundrySlices("NUMROWS", dimensionNameToLength) + + assertEquals(2, result.size) + + assertThat(result, hasItems("NUMROWS:0:1,NUMCELLS:0:82", "NUMROWS:1:2,NUMCELLS:0:82")) + + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByTilesDesiredTest.groovy ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByTilesDesiredTest.groovy b/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByTilesDesiredTest.groovy new file mode 100644 index 0000000..3b9f626 --- /dev/null +++ b/nexus-ingest/dataset-tiler/src/test/groovy/org/nasa/jpl/nexus/ingest/datatiler/SliceFileByTilesDesiredTest.groovy @@ -0,0 +1,33 @@ +/***************************************************************************** +* Copyright (c) 2016 Jet Propulsion Laboratory, +* California Institute of Technology. All rights reserved +*****************************************************************************/ +package org.nasa.jpl.nexus.ingest.datatiler + +import org.junit.Test + +import static junit.framework.Assert.assertEquals +import static org.hamcrest.CoreMatchers.hasItems +import static org.hamcrest.MatcherAssert.assertThat + +/** + * Created by greguska on 4/18/16. + */ +class SliceFileByTilesDesiredTest { + + @Test + public void testGenerateChunkBoundrySlices(){ + + def slicer = new SliceFileByTilesDesired() + + def chunksDesired = 5184 + def dimensionNameToLength = ['lat':17999, 'lon':36000] + + def result = slicer.generateChunkBoundrySlices(5184, dimensionNameToLength) + + assertEquals(chunksDesired + 72, result.size) + + assertThat(result, hasItems("lat:0:249,lon:0:500", "lat:0:249,lon:500:1000", "lat:17928:17999,lon:35500:36000")) + + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/src/test/resources/datasets/20150101090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.nc4 ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/src/test/resources/datasets/20150101090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.nc4 b/nexus-ingest/dataset-tiler/src/test/resources/datasets/20150101090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.nc4 new file mode 100644 index 0000000..ada9b07 Binary files /dev/null and b/nexus-ingest/dataset-tiler/src/test/resources/datasets/20150101090000-JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1.nc.nc4 differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/src/test/resources/datasets/CCMP_Wind_Analysis_20160101_V02.0_L3.0_RSS.split.nc ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/src/test/resources/datasets/CCMP_Wind_Analysis_20160101_V02.0_L3.0_RSS.split.nc b/nexus-ingest/dataset-tiler/src/test/resources/datasets/CCMP_Wind_Analysis_20160101_V02.0_L3.0_RSS.split.nc new file mode 100644 index 0000000..b7b491d Binary files /dev/null and b/nexus-ingest/dataset-tiler/src/test/resources/datasets/CCMP_Wind_Analysis_20160101_V02.0_L3.0_RSS.split.nc differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/src/test/resources/datasets/SMAP_L2B_SSS_00865_20150331T163144_R13080.split.h5 ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/src/test/resources/datasets/SMAP_L2B_SSS_00865_20150331T163144_R13080.split.h5 b/nexus-ingest/dataset-tiler/src/test/resources/datasets/SMAP_L2B_SSS_00865_20150331T163144_R13080.split.h5 new file mode 100644 index 0000000..956cbc5 Binary files /dev/null and b/nexus-ingest/dataset-tiler/src/test/resources/datasets/SMAP_L2B_SSS_00865_20150331T163144_R13080.split.h5 differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/dataset-tiler/src/test/resources/datasets/ascat_20130325_002100_metopb_02676_eps_o_coa_2101_ovw.l2.nc4 ---------------------------------------------------------------------- diff --git a/nexus-ingest/dataset-tiler/src/test/resources/datasets/ascat_20130325_002100_metopb_02676_eps_o_coa_2101_ovw.l2.nc4 b/nexus-ingest/dataset-tiler/src/test/resources/datasets/ascat_20130325_002100_metopb_02676_eps_o_coa_2101_ovw.l2.nc4 new file mode 100644 index 0000000..1214c24 Binary files /dev/null and b/nexus-ingest/dataset-tiler/src/test/resources/datasets/ascat_20130325_002100_metopb_02676_eps_o_coa_2101_ovw.l2.nc4 differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/.gitignore ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/.gitignore b/nexus-ingest/developer-box/.gitignore new file mode 100644 index 0000000..3dde3e5 --- /dev/null +++ b/nexus-ingest/developer-box/.gitignore @@ -0,0 +1,8 @@ +logs/ + +*.nc +*.h5 + +.listing +.DS_Store +.vagrant http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/README.md ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/README.md b/nexus-ingest/developer-box/README.md new file mode 100644 index 0000000..c1049a8 --- /dev/null +++ b/nexus-ingest/developer-box/README.md @@ -0,0 +1,103 @@ +# Reuirements + +* Vagrant (https://www.vagrantup.com/downloads.html) minimum version 1.8.1 + * The [vagrant-vbguest](https://github.com/dotless-de/vagrant-vbguest) plugin is also required. Once vagrant is installed, run `vagrant plugin install vagrant-vbguest` +* Apache Solr 5.3 with solr-nexustiles-core core running on localhost (see solr-nexustiles-core.zip) +* Apache Cassandra 2.2.3 with nexustiles keyspace running on localhost (see cassandra-ddl.sql) + +# Setup + +1. Clone this repo: https://github.com/dataplumber/nexus.git +2. CD into `nexus/nexus-ingest/developer-box` +3. Run the command `vagrant up` (this will take some time, around 10-20 minutes, itâs going to download a bunch of stuff) +4. Once completed, run `vagrant ssh` to log in to your newly minted centos virtual machine. +5. Once sshâed in, run `/vagrant/user-init.sh /home/vagrant` + * **NOTE** Part of this script will be cloning some git repos. In order to clone a JPL github repo you need a JPL github account, proper authorization for the repo, and an OAuth token. When running this script you will be prompted to enter an OAuth token. A token can be generated by going to https://github.com/settings/tokens +6. After that you should be able to go to http://localhost:9393/admin-ui/#/streams/definitions and youâll see a number of streams. +7. When changes are made in the repository, ssh into the vagrant box and run `./user-update.sh /home/vagrant` to deploy the new changes (FYI - this script re-deploys everything) + +## No Vagrant Setup + +1. Install JDK 8, Maven 3.3.9, Git (latest), and Anaconda2 4.0 +2. Create a conda environment and activate it + 1. conda install `libnetcdf` and `netcdf4` +3. Download the [Spring XD 1.3.1](http://repo.spring.io/libs-release/org/springframework/xd/spring-xd/1.3.1.RELEASE) distrobution zip + 1. Extract the zip to a folder `$springxd-home` + 2. Create directory `$springxd-home/spring-xd-1.3.1.RELEASE/xd/lib/none` + 3. Copy [xd-singlenode-logback.groovy](xd-singlenode-logback.groovy) to `$springxd-home/spring-xd-1.3.1.RELEASE/xd/config/xd-singlenode-logback.groovy` + 4. Start Spring XD in singlenode mode `nohup ./spring-xd-1.3.1.RELEASE/xd/bin/xd-singlenode --hadoopDistro none &` (logs will be in `$springxd-home/spring-xd-1.3.1.RELEASE/xd/logs/`) +4. Run `./user-init.sh $springxd-home $conda-environment-name` +5. When changes are made in the repository, run `./user-update.sh $springxd-home` to deploy the new changes (FYI - this script re-deploys everything) + +# Working with the Environment + +Now you have a centos virtualbox running on your local machine that has Spring XD installed and running in single node mode. + +## Ingesting Data + +Inside of the `developer-box` directory, you will notice a `data` directory with a number of sub-directories that coorespond to different datasets. If you copy a *.nc file into one of these directories and then deploy the dataset's corresponding stream, the *.nc files will begin to be ingested. + +### Datasets + +* MUR + * Data directory: `data/mur` + * Stream name: `ingest-mur` +* MUR Climatology + * Data directory: `data/mur-clim` + * Stream name: `ingest-mur` +* ASCATB L2 Coastal + * Data directory: `data/ascatb` + * Stream name: `ingest-ascatb-u` and `ingest-ascatb-v` +* AVHRR + * Data directory: `data/avhrr` + * Stream name: `ingest-avhrr` +* AVHRR Climatology + * Data directory: `data/avhrr-clim` + * Stream name: `ingest-avhrr-clim` +* MODIS Aqua L3 + * Data directory: `data/modis` + * Stream name: `ingest-modis` +* MODIS Aqua L3 Climatology + * Data directory: `data/modis-clim` + * Stream name: `ingest-modis-clim` +* TRMM + * Data directory: `data/trmm` + * Stream name: `ingest-trmm` +* SMAP (**WORK IN PROGRESS**) + * Data directory: `data/smap` + * Stream name: `ingest-smap-sss` + +## Interacting with the virtual machine + +* **SSH** You can ssh into the running virtual machine by running `vagrant ssh` from the `developer-box` directory +* **Halt** To stop the virtual machine, run `vagrant halt` from the `developer-box` directory +* **Start** To start the virtual machine, run `vagrant up` from the `developer-box` directory +* **Destroy** To destroy the virtual machine, run `vagrant destroy` from the `developer-box` directory + * **WARNING** This will remove all traces of the virtual machine. The next time you run `vagrant up` it will need to re-provision (i.e. download) all of the software again. + +The `/developer-box` directory on the host machine is automatically shared with the virtual machine. It can be accessed in `/vagrant` from the virtual machine. + +## Spring XD + +Spring XD is running inside the virtual machine in single-node mode. +It is installed in `/home/vagrant/spring-xd-1.3.1.RELEASE`. +Logs are written to `/vagrant/logs` which is also shared with the host machine and can be accessed from the host as `developer-box/logs` + +* **Stopping** To stop Spring XD run `ps -ef | grep xd-singlenode` to get the PID of the XD process and then `kill $PID` +* **Starting** To start Spring XD run `/vagrant/start-singlenode-at-boot.sh` + +### Opening a XD-Shell + +ssh into the vagrant machine and run `~/spring-xd-1.3.1.RELEASE/shell/bin/xd-shell` + +### Working with Streams + +* **Deploying** There are two ways to deploy a stream: + * Navigate to http://localhost:9393/admin-ui/#/streams/definitions and click on the `Deploy` button for the stream you want to deploy + * Open an XD Shell and run `stream deploy --name name-of-stream` +* **Un-Deploying** There are two ways to undeploy a stream: + * Navigate to http://localhost:9393/admin-ui/#/streams/definitions and click on the `Undeploy` button for the stream you want to deploy + * Open an XD Shell and run `stream undeploy --name name-of-stream` +* **Destroying** There are two was to destroy a stream: + * Navigate to http://localhost:9393/admin-ui/#/streams/definitions and click on the `Destroy` button for the stream you want to deploy + * Open an XD Shell and run `stream destroy --name name-of-stream` http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/Vagrantfile ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/Vagrantfile b/nexus-ingest/developer-box/Vagrantfile new file mode 100644 index 0000000..9966927 --- /dev/null +++ b/nexus-ingest/developer-box/Vagrantfile @@ -0,0 +1,74 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure(2) do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://atlas.hashicorp.com/search. + config.vm.box = "centos/7" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # config.vm.network "forwarded_port", guest: 80, host: 8080 + config.vm.network "forwarded_port", guest: 9393, host: 9393 + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network "private_network", ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + config.vm.synced_folder ".", "/home/vagrant/sync", type: "virtualbox" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + # config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # # Customize the amount of memory on the VM: + # vb.memory = "1024" + # end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies + # such as FTP and Heroku are also available. See the documentation at + # https://docs.vagrantup.com/v2/push/atlas.html for more information. + # config.push.define "atlas" do |push| + # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" + # end + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + # config.vm.provision "shell", inline: <<-SHELL + # sudo apt-get update + # sudo apt-get install -y apache2 + # SHELL + config.vm.provision :shell, path: "bootstrap.sh", privileged: false +end http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/bootstrap.sh ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/bootstrap.sh b/nexus-ingest/developer-box/bootstrap.sh new file mode 100644 index 0000000..c6e4137 --- /dev/null +++ b/nexus-ingest/developer-box/bootstrap.sh @@ -0,0 +1,61 @@ +# Install JDK +sudo yum -y install java-1.8.0-openjdk* + +# Install maven +echo "Downloading Maven" +curl "http://mirror.stjschools.org/public/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz"; -s -O +tar xvf apache-maven-3.3.9-bin.tar.gz +sudo mv apache-maven-3.3.9 /usr/local/apache-maven +cat >> /home/vagrant/.bashrc << END +# add for maven install +M2_HOME=/usr/local/apache-maven +M2=\$M2_HOME/bin +PATH=\$M2:\$PATH +END + +# Install nco tools +sudo yum -y install epel-release +sudo yum -y install nco-devel + +# Install git +sudo yum -y install git + +# Install and configure Anaconda +echo "Downloading Anaconda" +curl "https://repo.continuum.io/archive/Anaconda2-4.3.0-Linux-x86_64.sh"; -s -O + +chmod +x Anaconda2-4.3.0-Linux-x86_64.sh + +./Anaconda2-4.3.0-Linux-x86_64.sh -b -p /home/vagrant/anaconda2 + +cat >> /home/vagrant/.bashrc << END +# add for anaconda install +PATH=/home/vagrant/anaconda2/bin:\$PATH +END + +source /home/vagrant/.bashrc + +conda create -y --name nexus-xd-python-modules python +source activate nexus-xd-python-modules +conda install -y scipy=0.18.1 netcdf4 + + +# Install Spring XD and launch it + +sudo yum -y install unzip + +echo "Downloading Spring XD" +curl "http://repo.spring.io/libs-release/org/springframework/xd/spring-xd/1.3.1.RELEASE/spring-xd-1.3.1.RELEASE-dist.zip"; -s -O + +unzip spring-xd-1.3.1.RELEASE-dist.zip +mkdir spring-xd-1.3.1.RELEASE/xd/lib/none + +sudo ln -s /home/vagrant/sync /vagrant +cp /vagrant/xd-singlenode-logback.groovy spring-xd-1.3.1.RELEASE/xd/config/xd-singlenode-logback.groovy + +(crontab -l 2>/dev/null; echo "@reboot /vagrant/start-singlenode-at-boot.sh") | crontab - + +mkdir /vagrant/logs +pushd /home/vagrant +nohup ./spring-xd-1.3.1.RELEASE/xd/bin/xd-singlenode --hadoopDistro none > /vagrant/logs/xd-singlenode.log 2>&1 & +popd \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/cassandra-ddl.sql ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/cassandra-ddl.sql b/nexus-ingest/developer-box/cassandra-ddl.sql new file mode 100644 index 0000000..bfc486e --- /dev/null +++ b/nexus-ingest/developer-box/cassandra-ddl.sql @@ -0,0 +1,6 @@ +CREATE KEYSPACE IF NOT EXISTS nexustiles WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + +CREATE TABLE sea_surface_temp ( +tile_id uuid PRIMARY KEY, +tile_blob blob +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/data/ascatb/dataseturl.txt ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/data/ascatb/dataseturl.txt b/nexus-ingest/developer-box/data/ascatb/dataseturl.txt new file mode 100644 index 0000000..d274384 --- /dev/null +++ b/nexus-ingest/developer-box/data/ascatb/dataseturl.txt @@ -0,0 +1 @@ +https://podaac.jpl.nasa.gov/dataset/ASCATB-L2-Coastal http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/data/avhrr/dataseturl.txt ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/data/avhrr/dataseturl.txt b/nexus-ingest/developer-box/data/avhrr/dataseturl.txt new file mode 100644 index 0000000..70641d4 --- /dev/null +++ b/nexus-ingest/developer-box/data/avhrr/dataseturl.txt @@ -0,0 +1 @@ +ftp://podaac-ftp.jpl.nasa.gov/allData/ghrsst/data/GDS2/L4/GLOB/NCEI/AVHRR_OI/v2/ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/data/ccmp/dataseturl.txt ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/data/ccmp/dataseturl.txt b/nexus-ingest/developer-box/data/ccmp/dataseturl.txt new file mode 100644 index 0000000..a2cafec --- /dev/null +++ b/nexus-ingest/developer-box/data/ccmp/dataseturl.txt @@ -0,0 +1 @@ +ftp://ftp2.remss.com/ccmp/v02.0/ http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/data/grace/dataseturl.txt ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/data/grace/dataseturl.txt b/nexus-ingest/developer-box/data/grace/dataseturl.txt new file mode 100644 index 0000000..c4cdb3b --- /dev/null +++ b/nexus-ingest/developer-box/data/grace/dataseturl.txt @@ -0,0 +1 @@ +https://podaac.jpl.nasa.gov/dataset/TELLUS_GRACE_MASCON_GRID_RL05_V2 http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/data/measures_alt/dataseturl.txt ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/data/measures_alt/dataseturl.txt b/nexus-ingest/developer-box/data/measures_alt/dataseturl.txt new file mode 100644 index 0000000..fd3bfde --- /dev/null +++ b/nexus-ingest/developer-box/data/measures_alt/dataseturl.txt @@ -0,0 +1 @@ +ftp://podaac-ftp.jpl.nasa.gov/allData/merged_alt/L4/ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/data/modis/dataseturl.txt ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/data/modis/dataseturl.txt b/nexus-ingest/developer-box/data/modis/dataseturl.txt new file mode 100644 index 0000000..acac6bd --- /dev/null +++ b/nexus-ingest/developer-box/data/modis/dataseturl.txt @@ -0,0 +1 @@ +http://podaac.jpl.nasa.gov/dataset/MODIS_AQUA_L3_SST_THERMAL_DAILY_4KM_DAYTIME_V2014.0 http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/data/modis_aqua_chl/dataseturl.txt ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/data/modis_aqua_chl/dataseturl.txt b/nexus-ingest/developer-box/data/modis_aqua_chl/dataseturl.txt new file mode 100644 index 0000000..02e0534 --- /dev/null +++ b/nexus-ingest/developer-box/data/modis_aqua_chl/dataseturl.txt @@ -0,0 +1 @@ +https://oceandata.sci.gsfc.nasa.gov/MODIS-Aqua/Mapped/Daily/4km/chlor_a \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/data/mur/dataseturl.txt ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/data/mur/dataseturl.txt b/nexus-ingest/developer-box/data/mur/dataseturl.txt new file mode 100644 index 0000000..20d38a3 --- /dev/null +++ b/nexus-ingest/developer-box/data/mur/dataseturl.txt @@ -0,0 +1 @@ +https://podaac.jpl.nasa.gov/dataset/MUR-JPL-L4-GLOB-v4.1 http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/data/smap/dataseturl.txt ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/data/smap/dataseturl.txt b/nexus-ingest/developer-box/data/smap/dataseturl.txt new file mode 100644 index 0000000..8e99c9f --- /dev/null +++ b/nexus-ingest/developer-box/data/smap/dataseturl.txt @@ -0,0 +1 @@ +ftp://sealion.jpl.nasa.gov/pub/outgoing/smap/v2.0/L2B \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/data/smapl3/dataseturl.txt ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/data/smapl3/dataseturl.txt b/nexus-ingest/developer-box/data/smapl3/dataseturl.txt new file mode 100644 index 0000000..2b4993d --- /dev/null +++ b/nexus-ingest/developer-box/data/smapl3/dataseturl.txt @@ -0,0 +1 @@ +ftp://podaac-ftp.jpl.nasa.gov/allData/smap/L3/RSS/V2/monthly/SCI \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/data/trmm/dataseturl.txt ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/data/trmm/dataseturl.txt b/nexus-ingest/developer-box/data/trmm/dataseturl.txt new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/solr-nexustiles-core.zip ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/solr-nexustiles-core.zip b/nexus-ingest/developer-box/solr-nexustiles-core.zip new file mode 100644 index 0000000..c510af5 Binary files /dev/null and b/nexus-ingest/developer-box/solr-nexustiles-core.zip differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/start-singlenode-at-boot.sh ---------------------------------------------------------------------- diff --git a/nexus-ingest/developer-box/start-singlenode-at-boot.sh b/nexus-ingest/developer-box/start-singlenode-at-boot.sh new file mode 100755 index 0000000..84a6aa3 --- /dev/null +++ b/nexus-ingest/developer-box/start-singlenode-at-boot.sh @@ -0,0 +1,4 @@ +source activate nexus-xd-python-modules +pushd /home/vagrant +nohup ./spring-xd-1.3.1.RELEASE/xd/bin/xd-singlenode --hadoopDistro none > /vagrant/logs/xd-singlenode.log 2>&1 & +popd
