This is an automated email from the ASF dual-hosted git repository. fgreg pushed a commit to branch SDAP-48 in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexusproto.git
commit 9be5c61545a8d3ffb9111f7f760ee968adc7d3c5 Author: Frank Greguska <[email protected]> AuthorDate: Thu Oct 19 10:43:58 2017 -0700 initial commit --- .gitignore | 35 +++++ README.md | 24 ++++ build.gradle | 122 +++++++++++++++++ gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 54711 bytes gradle/wrapper/gradle-wrapper.properties | 6 + gradlew | 172 ++++++++++++++++++++++++ src/main/proto/DataTile.proto | 66 +++++++++ src/main/python/ningesterproto/__init__.py | 6 + src/main/python/ningesterproto/serialization.py | 40 ++++++ src/main/python/ningesterproto/setup.py | 31 +++++ 10 files changed, 502 insertions(+) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ab9b2f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,35 @@ +.gradle/ +.idea/ +gradlew.bat + +distrobution/ + +gen/ + +build/* +!build/reports + +build/reports/* +!build/reports/license +!build/reports/project + +nexusproto.egg-info/ + +.DS_Store +*.log + +#Idea files +*.iml +*.ipr +*.iws + +*.class + +# Package Files # +*.war +*.ear + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +gradle.properties diff --git a/README.md b/README.md new file mode 100644 index 0000000..34bf7c7 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +# nexus-messages + +This project contains the [protobuf](https://developers.google.com/protocol-buffers/) definition for a NexusTile. By compiling the protobuf specification, both Java and Python objects are generated. + +# Developer Installation + +1. Run `./gradlew clean build install` + +2. cd into `/build/python/nexusproto` + +3. Setup a separate conda env or activate an existing one + + ```` + conda create --name nexus-messages python + source activate nexus-messages + ```` + +4. Install Conda dependencies + + ```` + conda install numpy + ```` + +5. Run `python setup.py install` diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..dc0bb41 --- /dev/null +++ b/build.gradle @@ -0,0 +1,122 @@ +import java.nio.file.* +import java.nio.file.attribute.BasicFileAttributes + +buildscript { + repositories { + mavenCentral() + jcenter() + } + dependencies { + classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.3' + } +} + +repositories { + mavenCentral() +} + + +apply plugin: 'java' +apply plugin: 'com.google.protobuf' +apply plugin: 'maven' +apply plugin: 'maven-publish' +apply plugin: 'project-report' + + +group = 'gov.nasa.jpl.nexus.ningester' +version = '1.0.0.RELEASE' + +ext { + genDirectory = "$projectDir/gen" + distDirectory = "$projectDir/distrobution" + pythonBuildDirPath = "${file(buildDir.path + '/python/ningesterproto').path}" +} + +protobuf { + + generatedFilesBaseDir = genDirectory + + // Configure the protoc executable + protoc { + // Download from repositories + artifact = 'com.google.protobuf:protoc:3.4.0' + } + + plugins { + // Define a plugin with name 'grpc' + grpc { + artifact = 'io.grpc:protoc-gen-grpc-java:1.7.0' + } + } + + generateProtoTasks { + all().each { task -> + task.builtins { + python { + outputSubDir = 'python' + } + } + } + } + +} + +task writeNewPom { + doLast { + pom {}.writeTo(file(buildDir.path + "/poms/${project.name}-${project.version}.xml")) + } +} + +publishing { + publications { + mavenJava(MavenPublication) { + from components.java + } + } +} + +assemble.doLast { + File pythonbuilddir = file(pythonBuildDirPath) + + File pythonsource = file('src/main/python') + + Files.walkFileTree(pythonsource.toPath(), new SimpleFileVisitor<Path>() { + @Override + public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException { + Files.createDirectories(pythonbuilddir.toPath().resolve(pythonsource.toPath().relativize(dir))) + return FileVisitResult.CONTINUE + } + + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + Files.copy(file, pythonbuilddir.toPath().resolve(pythonsource.toPath().relativize(file)), StandardCopyOption.REPLACE_EXISTING) + return FileVisitResult.CONTINUE + } + }) + + Files.move(file(pythonbuilddir.path + '/ningesterproto/setup.py').toPath(), file(pythonbuilddir.path + '/setup.py').toPath(), StandardCopyOption.REPLACE_EXISTING) + + File generatedPython = file("$genDirectory/main/python/DataTile_pb2.py") + + Files.copy(generatedPython.toPath(), file(pythonbuilddir.path + '/ningesterproto/DataTile_pb2.py').toPath(), StandardCopyOption.REPLACE_EXISTING) +} + +task tarPython(type: Tar, dependsOn: [assemble]) { + destinationDir = file("distrobution") + archiveName = 'ningesterproto.tar.gz' + compression = Compression.GZIP + from(file(buildDir.path + '/python')) { + include '**/*' + } +} + +clean.doLast { + file(genDirectory).deleteDir() + file(distDirectory).deleteDir() +} + +dependencies { + + compile 'com.google.protobuf:protobuf-java:3.4.0' + +} \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..d1645de Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..74a41c0 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Tue Oct 17 11:06:47 PDT 2017 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-3.5.1-bin.zip diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..4453cce --- /dev/null +++ b/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save ( ) { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/src/main/proto/DataTile.proto b/src/main/proto/DataTile.proto new file mode 100644 index 0000000..b947b11 --- /dev/null +++ b/src/main/proto/DataTile.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; + +package gov.nasa.jpl.nexus.ningester.protobuf; +option java_package = "gov.nasa.jpl.nexus.ningester.protobuf"; +option java_multiple_files = true; + +service ProtoTileService { + + rpc ToTile(stream TileSpecification) returns (stream Tile) {} + rpc ProcessTile(stream Tile) returns (stream Tile){} + +} + +message TileSpecification{ + + message Slice { + oneof name_or_position { + string dimension_name = 1; + int32 dimension_position = 2; + } + int64 start = 3; + int64 stop = 4; + int64 step = 5; + } + + repeated Slice slices = 1; +} + +message Tile{ + + string tile_id = 1; + TileSpecification section_spec = 2; + string dataset_name = 3; + string granule = 4; + string dataset_uuid = 5; + string variable_name = 6; + repeated string metadata_names = 7; + + message BBox{ + float lat_min = 1; + float lat_max = 2; + float lon_min = 3; + float lon_max = 4; + } + BBox bbox = 8; + + message DataStats{ + float min = 1; + float max = 2; + float mean = 3; + int64 count = 4; + + int64 min_time = 5; + int64 max_time = 6; + + float std_dev = 7; + float sum = 8; + float median = 9; + float variance = 10; + float skewness = 11; + float kurtosis = 12; + } + DataStats stats = 9; + + bytes data = 10; +} \ No newline at end of file diff --git a/src/main/python/ningesterproto/__init__.py b/src/main/python/ningesterproto/__init__.py new file mode 100644 index 0000000..f7fad5c --- /dev/null +++ b/src/main/python/ningesterproto/__init__.py @@ -0,0 +1,6 @@ +""" +Copyright (c) 2017 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" + +from serialization import * diff --git a/src/main/python/ningesterproto/serialization.py b/src/main/python/ningesterproto/serialization.py new file mode 100644 index 0000000..64fe207 --- /dev/null +++ b/src/main/python/ningesterproto/serialization.py @@ -0,0 +1,40 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +import StringIO + +import numpy + +import nexusproto.NexusContent_pb2 as nexusproto + + +def from_shaped_array(shaped_array): + memfile = StringIO.StringIO() + memfile.write(shaped_array.array_data) + memfile.seek(0) + data_array = numpy.load(memfile) + memfile.close() + + return data_array + + +def to_shaped_array(data_array): + shaped_array = nexusproto.ShapedArray() + + shaped_array.shape.extend([dimension_size for dimension_size in data_array.shape]) + shaped_array.dtype = str(data_array.dtype) + + memfile = StringIO.StringIO() + numpy.save(memfile, data_array) + shaped_array.array_data = memfile.getvalue() + memfile.close() + + return shaped_array + +def to_metadata(name, data_array): + metadata = nexusproto.MetaData() + metadata.name = name + metadata.meta_data.CopyFrom(to_shaped_array(data_array)) + + return metadata diff --git a/src/main/python/ningesterproto/setup.py b/src/main/python/ningesterproto/setup.py new file mode 100644 index 0000000..b1bcaec --- /dev/null +++ b/src/main/python/ningesterproto/setup.py @@ -0,0 +1,31 @@ +""" +Copyright (c) 2016 Jet Propulsion Laboratory, +California Institute of Technology. All rights reserved +""" +from setuptools import setup + +__version__ = '0.1' + +setup( + name='ningesterproto', + version=__version__, + url="https://github.com/aist-oceanworks";, + + author="Team Nexus", + + description="Protobufs used while ingesting NEXUS tiles.", + + packages=['ningesterproto'], + platforms='any', + + install_requires=[ + 'protobuf' + ], + + classifiers=[ + 'Development Status :: 1 - Pre-Alpha', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2.7', + ] +)
