initial commit
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/commit/03f6c22e Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/tree/03f6c22e Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/diff/03f6c22e Branch: refs/heads/master Commit: 03f6c22ecda54d8f8ec9be83cd44e44e7cc606f9 Parents: 5be0e5d Author: Frank Greguska <[email protected]> Authored: Thu Oct 12 17:36:39 2017 -0700 Committer: Frank Greguska <[email protected]> Committed: Thu Oct 12 17:39:28 2017 -0700 ---------------------------------------------------------------------- .gitignore | 25 +++ build.gradle | 44 +++++ gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 54711 bytes gradle/wrapper/gradle-wrapper.properties | 6 + gradlew | 172 +++++++++++++++++++ gradlew.bat | 84 +++++++++ .../nexus/ningester/NingesterApplication.class | Bin 0 -> 743 bytes .../ningester/configuration/BatchConfig.class | Bin 0 -> 5808 bytes .../configuration/DatasourceConfiguration.class | Bin 0 -> 1322 bytes .../nexus/ningester/datatiler/FileSlicer.class | Bin 0 -> 275 bytes .../ningester/datatiler/NetCDFItemReader.class | Bin 0 -> 1698 bytes .../datatiler/SliceFileByTilesDesired.class | Bin 0 -> 7374 bytes out/production/resources/application.properties | 0 .../ningester/NingesterApplicationTests.class | Bin 0 -> 661 bytes .../datatiler/SliceFileByTilesDesiredTest.class | Bin 0 -> 2990 bytes .../nexus/ningester/NingesterApplication.java | 12 ++ .../ningester/configuration/BatchConfig.java | 75 ++++++++ .../configuration/DatasourceConfiguration.java | 22 +++ .../nexus/ningester/datatiler/FileSlicer.java | 15 ++ .../ningester/datatiler/NetCDFItemReader.java | 26 +++ .../datatiler/SliceFileByTilesDesired.java | 77 +++++++++ src/main/resources/application.properties | 0 .../ningester/NingesterApplicationTests.java | 16 ++ .../datatiler/SliceFileByTilesDesiredTest.java | 95 ++++++++++ 24 files changed, 669 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c9f7010 --- /dev/null +++ b/.gitignore @@ -0,0 +1,25 @@ +.gradle +/build/ +!gradle/wrapper/gradle-wrapper.jar + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +nbproject/private/ +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..c2076a0 --- /dev/null +++ b/build.gradle @@ -0,0 +1,44 @@ +buildscript { + ext { + springBootVersion = '1.5.7.RELEASE' + } + repositories { + mavenCentral() + } + dependencies { + classpath("org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}") + } +} + +apply plugin: 'java' +apply plugin: 'eclipse' +apply plugin: 'org.springframework.boot' + +group = 'gov.nasa.jpl.nexus.ningester' +version = '0.0.1-SNAPSHOT' +sourceCompatibility = 1.8 + +repositories { + mavenCentral() + maven { + url "https://artifacts.unidata.ucar.edu/content/repositories/unidata-releases/"; + } + mavenLocal() +} + +ext{ + nexusMessagesVersion = "1.0.0.RELEASE" + netcdfJavaVersion = '4.6.9' + guavaVersion = "23.2-jre" +} + + +dependencies { + compile('org.springframework.boot:spring-boot-starter-batch') + compile("com.h2database:h2") + compile("org.nasa.jpl.nexus:nexus-messages:$nexusMessagesVersion") + compile("edu.ucar:cdm:${netcdfJavaVersion}") + compile group: 'com.google.guava', name: 'guava' + + testCompile('org.springframework.boot:spring-boot-starter-test') +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/gradle/wrapper/gradle-wrapper.jar ---------------------------------------------------------------------- diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..391f298 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/gradle/wrapper/gradle-wrapper.properties ---------------------------------------------------------------------- diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..e1d4367 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Thu Oct 12 10:22:24 PDT 2017 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-3.5.1-bin.zip http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/gradlew ---------------------------------------------------------------------- diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..4453cce --- /dev/null +++ b/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save ( ) { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/gradlew.bat ---------------------------------------------------------------------- diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..f955316 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/out/production/classes/gov/nasa/jpl/nexus/ningester/NingesterApplication.class ---------------------------------------------------------------------- diff --git a/out/production/classes/gov/nasa/jpl/nexus/ningester/NingesterApplication.class b/out/production/classes/gov/nasa/jpl/nexus/ningester/NingesterApplication.class new file mode 100644 index 0000000..1c0727d Binary files /dev/null and b/out/production/classes/gov/nasa/jpl/nexus/ningester/NingesterApplication.class differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/out/production/classes/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.class ---------------------------------------------------------------------- diff --git a/out/production/classes/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.class b/out/production/classes/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.class new file mode 100644 index 0000000..ae0a8d4 Binary files /dev/null and b/out/production/classes/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.class differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/out/production/classes/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfiguration.class ---------------------------------------------------------------------- diff --git a/out/production/classes/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfiguration.class b/out/production/classes/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfiguration.class new file mode 100644 index 0000000..a52bf80 Binary files /dev/null and b/out/production/classes/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfiguration.class differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/out/production/classes/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.class ---------------------------------------------------------------------- diff --git a/out/production/classes/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.class b/out/production/classes/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.class new file mode 100644 index 0000000..37b3eb9 Binary files /dev/null and b/out/production/classes/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.class differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/out/production/classes/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.class ---------------------------------------------------------------------- diff --git a/out/production/classes/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.class b/out/production/classes/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.class new file mode 100644 index 0000000..6b7d331 Binary files /dev/null and b/out/production/classes/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.class differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/out/production/classes/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.class ---------------------------------------------------------------------- diff --git a/out/production/classes/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.class b/out/production/classes/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.class new file mode 100644 index 0000000..119beeb Binary files /dev/null and b/out/production/classes/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.class differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/out/production/resources/application.properties ---------------------------------------------------------------------- diff --git a/out/production/resources/application.properties b/out/production/resources/application.properties new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/out/test/classes/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.class ---------------------------------------------------------------------- diff --git a/out/test/classes/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.class b/out/test/classes/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.class new file mode 100644 index 0000000..7912a59 Binary files /dev/null and b/out/test/classes/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.class differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/out/test/classes/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.class ---------------------------------------------------------------------- diff --git a/out/test/classes/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.class b/out/test/classes/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.class new file mode 100644 index 0000000..2832512 Binary files /dev/null and b/out/test/classes/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.class differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/src/main/java/gov/nasa/jpl/nexus/ningester/NingesterApplication.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/NingesterApplication.java b/src/main/java/gov/nasa/jpl/nexus/ningester/NingesterApplication.java new file mode 100644 index 0000000..5945596 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/NingesterApplication.java @@ -0,0 +1,12 @@ +package gov.nasa.jpl.nexus.ningester; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class NingesterApplication { + + public static void main(String[] args) { + SpringApplication.run(NingesterApplication.class, args); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java new file mode 100644 index 0000000..7febcdf --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/BatchConfig.java @@ -0,0 +1,75 @@ +package gov.nasa.jpl.nexus.ningester.configuration; + +import gov.nasa.jpl.nexus.ningester.datatiler.FileSlicer; +import gov.nasa.jpl.nexus.ningester.datatiler.SliceFileByTilesDesired; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.JobScope; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.adapter.ItemWriterAdapter; +import org.springframework.batch.item.support.ListItemReader; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.Resource; +import org.springframework.core.io.ResourceLoader; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +@Configuration +@EnableBatchProcessing +public class BatchConfig { + + @Autowired + private JobBuilderFactory jobs; + + @Autowired + private StepBuilderFactory steps; + + @Bean + public Job job(@Qualifier("step1") Step step1) { + return jobs.get("testNc4").start(step1).build(); + } + + @Bean + @JobScope + protected Resource granule(ResourceLoader resourceLoader, @Value("#{jobParameters['granule']}") String granuleLocation){ + return resourceLoader.getResource(granuleLocation); + } + + @Bean + @JobScope + protected List<String> tileSpecifications(Resource granule) throws IOException { + SliceFileByTilesDesired fileSlicer = new SliceFileByTilesDesired(); + fileSlicer.setDimensions(Arrays.asList("lat", "lon")); + fileSlicer.setTilesDesired(5184); + return fileSlicer.generateSlices(granule.getFile()); + } + + @Bean + @JobScope + protected ItemReader<String> reader(List<String> tileSpecifications) { + return new ListItemReader<>(tileSpecifications); + } + + @Bean + protected ItemWriter<String> writer() { + ItemWriterAdapter<String> writer = new ItemWriterAdapter<>(); + writer.setTargetMethod("println"); + writer.setTargetObject(System.out); + return writer; + } + + @Bean + protected Step step1(ItemReader<String> reader, ItemWriter<String> writer) { + return steps.get("step1").<String, String>chunk(10).reader(reader).writer(writer).build(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfiguration.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfiguration.java b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfiguration.java new file mode 100644 index 0000000..eb3e498 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/configuration/DatasourceConfiguration.java @@ -0,0 +1,22 @@ +package gov.nasa.jpl.nexus.ningester.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import javax.sql.DataSource; + +@Configuration +public class DatasourceConfiguration { + + @Bean + @Profile("embedded") + public DataSource dataSource(){ + EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder(); + return builder + .setType(EmbeddedDatabaseType.H2) + .build(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.java new file mode 100644 index 0000000..44c661c --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/FileSlicer.java @@ -0,0 +1,15 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.datatiler; + +import java.io.File; +import java.util.List; + +public interface FileSlicer { + + List<String> generateSlices(File inputfile); + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java new file mode 100644 index 0000000..f9d2e49 --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/NetCDFItemReader.java @@ -0,0 +1,26 @@ +package gov.nasa.jpl.nexus.ningester.datatiler; + +import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent; +import org.springframework.batch.item.*; + +public class NetCDFItemReader implements ItemReader<NexusContent.NexusTile>, ItemStream { + @Override + public NexusContent.NexusTile read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException { + return null; + } + + @Override + public void open(ExecutionContext executionContext) throws ItemStreamException { + + } + + @Override + public void update(ExecutionContext executionContext) throws ItemStreamException { + + } + + @Override + public void close() throws ItemStreamException { + + } +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java ---------------------------------------------------------------------- diff --git a/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java new file mode 100644 index 0000000..ba8629a --- /dev/null +++ b/src/main/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesired.java @@ -0,0 +1,77 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ +package gov.nasa.jpl.nexus.ningester.datatiler; + +import com.google.common.collect.Sets; +import ucar.nc2.Dimension; +import ucar.nc2.dataset.NetcdfDataset; + +import java.io.File; +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; + +public class SliceFileByTilesDesired implements FileSlicer { + + private Integer tilesDesired; + private List<String> dimensions; + + public void setTilesDesired(Integer desired) { + this.tilesDesired = desired; + } + + public void setDimensions(List<String> dims) { + this.dimensions = dims; + } + + @Override + public List<String> generateSlices(File inputfile) { + + Map<String, Integer> dimensionNameToLength; + try (NetcdfDataset ds = NetcdfDataset.openDataset(inputfile.getAbsolutePath())) { + + + dimensionNameToLength = ds.getDimensions().stream() + .filter(dimension -> this.dimensions.contains(dimension.getShortName())) + .collect(Collectors.toMap(Dimension::getShortName, Dimension::getLength, + (v1,v2) ->{ throw new RuntimeException(String.format("Duplicate key for values %s and %s", v1, v2));}, + TreeMap::new)); + } catch (IOException e) { + throw new RuntimeException("Error reading netcdf file", e); + } + + return generateChunkBoundrySlices(tilesDesired, dimensionNameToLength); + + } + + List<String> generateChunkBoundrySlices(Integer tilesDesired, Map<String, Integer> dimensionNameToLength) { + + List<Set<String>> dimensionBounds = dimensionNameToLength.entrySet().stream() + .map(stringIntegerEntry -> { + String dimensionName = stringIntegerEntry.getKey(); + Integer lengthOfDimension = stringIntegerEntry.getValue(); + Integer stepSize = calculateStepSize(stringIntegerEntry.getValue(), tilesDesired, dimensionNameToLength.size()); + Set<String> bounds = new LinkedHashSet<>(); + for(int i = 0; i < lengthOfDimension; i += stepSize){ + bounds.add( + dimensionName + ":" + + i + ":" + + (i + stepSize >= lengthOfDimension ? lengthOfDimension : i + stepSize)); + } + return bounds; + }).collect(Collectors.toList()); + + return Sets.cartesianProduct(dimensionBounds) + .stream() + .map(tileSpecAsList -> tileSpecAsList.stream().collect(Collectors.joining(","))) + .collect(Collectors.toList()); + + } + + private Integer calculateStepSize(Integer lengthOfDimension, Integer chunksDesired, Integer numberOfDimensions) { + return new Double(Math.floor(lengthOfDimension / (Math.pow(chunksDesired, (1.0 / numberOfDimensions))))).intValue(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/src/main/resources/application.properties ---------------------------------------------------------------------- diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/src/test/java/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.java b/src/test/java/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.java new file mode 100644 index 0000000..6e3e94e --- /dev/null +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/NingesterApplicationTests.java @@ -0,0 +1,16 @@ +package gov.nasa.jpl.nexus.ningester; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +public class NingesterApplicationTests { + + @Test + public void contextLoads() { + } + +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-ningester/blob/03f6c22e/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.java b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.java new file mode 100644 index 0000000..2d6dbdb --- /dev/null +++ b/src/test/java/gov/nasa/jpl/nexus/ningester/datatiler/SliceFileByTilesDesiredTest.java @@ -0,0 +1,95 @@ +/***************************************************************************** + * Copyright (c) 2017 Jet Propulsion Laboratory, + * California Institute of Technology. All rights reserved + *****************************************************************************/ + +package gov.nasa.jpl.nexus.ningester.datatiler; + +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static junit.framework.Assert.assertEquals; +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; + +public class SliceFileByTilesDesiredTest { + + @Test + public void testGenerateChunkBoundrySlicesWithDivisibileTiles() { + + SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); + + Integer tilesDesired = 4; + + Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>(); + dimensionNameToLength.put("lat", 8); + dimensionNameToLength.put("lon", 8); + + List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength); + + assertEquals(tilesDesired.intValue(), result.size()); + + String[] expected = new String[]{ + "lat:0:4,lon:0:4", + "lat:0:4,lon:4:8", + "lat:4:8,lon:0:4", + "lat:4:8,lon:4:8"}; + assertThat(result, containsInAnyOrder(expected)); + assertThat(result, contains(expected)); + + } + + @Test + public void testGenerateChunkBoundrySlicesWithNonDivisibileTiles() { + + SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); + + Integer tilesDesired = 5; + + Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>(); + dimensionNameToLength.put("lat", 8); + dimensionNameToLength.put("lon", 8); + + List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength); + + assertEquals(9, result.size()); + + String[] expected = new String[]{ + "lat:0:3,lon:0:3", + "lat:0:3,lon:3:6", + "lat:0:3,lon:6:8", + "lat:3:6,lon:0:3", + "lat:3:6,lon:3:6", + "lat:3:6,lon:6:8", + "lat:6:8,lon:0:3", + "lat:6:8,lon:3:6", + "lat:6:8,lon:6:8"}; + assertThat(result, containsInAnyOrder(expected)); + assertThat(result, contains(expected)); + + } + + @Test + public void testGenerateChunkBoundrySlicesWithMurDimensions() { + + SliceFileByTilesDesired slicer = new SliceFileByTilesDesired(); + + Integer tilesDesired = 5184; + + Map<String, Integer> dimensionNameToLength = new LinkedHashMap<>(); + dimensionNameToLength.put("lat", 17999); + dimensionNameToLength.put("lon", 36000); + + List<String> result = slicer.generateChunkBoundrySlices(tilesDesired, dimensionNameToLength); + + assertEquals(tilesDesired + 72, result.size()); + + assertThat(result, hasItems("lat:0:249,lon:0:500", "lat:0:249,lon:500:1000", "lat:17928:17999,lon:35500:36000")); + + } +}
