updated build
Project: http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/commit/135cf8a0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/tree/135cf8a0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/diff/135cf8a0 Branch: refs/heads/master Commit: 135cf8a0d469926cf729107318b081d0fd66f198 Parents: 9be5c61 Author: Frank Greguska <[email protected]> Authored: Mon Jan 22 14:34:28 2018 -0800 Committer: Frank Greguska <[email protected]> Committed: Mon Jan 22 14:34:28 2018 -0800 ---------------------------------------------------------------------- .gitignore | 234 +++++++++++++++++-- .idea/.name | 1 + .idea/compiler.xml | 11 + .idea/encodings.xml | 6 + .idea/misc.xml | 6 + .idea/modules.xml | 10 + .idea/modules/nexusproto_main.iml | 14 ++ .idea/modules/nexusproto_test.iml | 16 ++ .idea/vcs.xml | 6 + build.gradle | 65 ++++-- gradle.properties | 3 + gradle/wrapper/gradle-wrapper.jar | Bin 54711 -> 54711 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew.bat | 84 +++++++ nexusproto.iml | 13 ++ src/main/proto/DataTile.proto | 103 ++++++-- src/main/python/nexusproto/__init__.py | 16 ++ src/main/python/nexusproto/serialization.py | 52 +++++ src/main/python/nexusproto/setup.py | 41 ++++ src/main/python/ningesterproto/__init__.py | 6 - src/main/python/ningesterproto/serialization.py | 40 ---- src/main/python/ningesterproto/setup.py | 31 --- 22 files changed, 623 insertions(+), 137 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index ab9b2f9..f350e37 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,3 @@ -.gradle/ -.idea/ -gradlew.bat - -distrobution/ - -gen/ - build/* !build/reports @@ -13,23 +5,237 @@ build/reports/* !build/reports/license !build/reports/project -nexusproto.egg-info/ +distrobution/ -.DS_Store +gen/ + + +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: *.log +.static_storage/ +.media/ +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ -#Idea files -*.iml -*.ipr +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: *.iws +## Plugin-specific files: + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties +### VirtualEnv template +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +[Bb]in +[Ii]nclude +[Ll]ib +[Ll]ib64 +[Ll]ocal +[Ss]cripts +pyvenv.cfg +pip-selfcheck.json +### macOS template +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk +### Java template +# Compiled class file *.class +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + # Package Files # +*.jar *.war *.ear +*.zip +*.tar.gz +*.rar # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* -gradle.properties +### Gradle template +.gradle +/build/ + +# Ignore Gradle GUI config +gradle-app.setting + +# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +!gradle-wrapper.jar + +# Cache of project +.gradletasknamecache + +# # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# gradle/wrapper/gradle-wrapper.properties http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/.idea/.name ---------------------------------------------------------------------- diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000..4e9f175 --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +nexusproto \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/.idea/compiler.xml ---------------------------------------------------------------------- diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..a7cf469 --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="CompilerConfiguration"> + <bytecodeTargetLevel> + <module name="nexusproto_main" target="1.8" /> + <module name="nexusproto_test" target="1.8" /> + <module name="ningester-protobuf_main" target="1.8" /> + <module name="ningester-protobuf_test" target="1.8" /> + </bytecodeTargetLevel> + </component> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/.idea/encodings.xml ---------------------------------------------------------------------- diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..97626ba --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="Encoding"> + <file url="PROJECT" charset="UTF-8" /> + </component> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/.idea/misc.xml ---------------------------------------------------------------------- diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..44ae3bd --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="false" project-jdk-name="1.8" project-jdk-type="JavaSDK"> + <output url="file://$PROJECT_DIR$/classes" /> + </component> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/.idea/modules.xml ---------------------------------------------------------------------- diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..a7e194b --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,10 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="ProjectModuleManager"> + <modules> + <module fileurl="file://$PROJECT_DIR$/nexusproto.iml" filepath="$PROJECT_DIR$/nexusproto.iml" /> + <module fileurl="file://$PROJECT_DIR$/.idea/modules/nexusproto_main.iml" filepath="$PROJECT_DIR$/.idea/modules/nexusproto_main.iml" group="nexusproto" /> + <module fileurl="file://$PROJECT_DIR$/.idea/modules/nexusproto_test.iml" filepath="$PROJECT_DIR$/.idea/modules/nexusproto_test.iml" group="nexusproto" /> + </modules> + </component> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/.idea/modules/nexusproto_main.iml ---------------------------------------------------------------------- diff --git a/.idea/modules/nexusproto_main.iml b/.idea/modules/nexusproto_main.iml new file mode 100644 index 0000000..403cc46 --- /dev/null +++ b/.idea/modules/nexusproto_main.iml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<module external.linked.project.id="nexusproto:main" external.linked.project.path="$MODULE_DIR$/../.." external.root.project.path="$MODULE_DIR$/../.." external.system.id="GRADLE" external.system.module.group="'org.apache.sdap'" external.system.module.type="sourceSet" external.system.module.version="1.0.1-SNAPSHOT" type="JAVA_MODULE" version="4"> + <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8"> + <output url="file://$MODULE_DIR$/../../out/production/classes" /> + <exclude-output /> + <content url="file://$MODULE_DIR$/../../src/main"> + <sourceFolder url="file://$MODULE_DIR$/../../src/main/java" isTestSource="false" /> + <sourceFolder url="file://$MODULE_DIR$/../../src/main/resources" type="java-resource" /> + </content> + <orderEntry type="inheritedJdk" /> + <orderEntry type="sourceFolder" forTests="false" /> + <orderEntry type="library" name="Gradle: com.google.protobuf:protobuf-java:3.4.0" level="project" /> + </component> +</module> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/.idea/modules/nexusproto_test.iml ---------------------------------------------------------------------- diff --git a/.idea/modules/nexusproto_test.iml b/.idea/modules/nexusproto_test.iml new file mode 100644 index 0000000..c80b72c --- /dev/null +++ b/.idea/modules/nexusproto_test.iml @@ -0,0 +1,16 @@ +<?xml version="1.0" encoding="UTF-8"?> +<module external.linked.project.id="nexusproto:test" external.linked.project.path="$MODULE_DIR$/../.." external.root.project.path="$MODULE_DIR$/../.." external.system.id="GRADLE" external.system.module.group="'org.apache.sdap'" external.system.module.type="sourceSet" external.system.module.version="1.0.1-SNAPSHOT" type="JAVA_MODULE" version="4"> + <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8"> + <output-test url="file://$MODULE_DIR$/../../out/test/classes" /> + <exclude-output /> + <content url="file://$MODULE_DIR$/../../src/test"> + <sourceFolder url="file://$MODULE_DIR$/../../src/test/java" isTestSource="true" /> + <sourceFolder url="file://$MODULE_DIR$/../../src/test/resources" type="java-test-resource" /> + </content> + <orderEntry type="inheritedJdk" /> + <orderEntry type="sourceFolder" forTests="false" /> + <orderEntry type="module" module-name="nexusproto_main" /> + <orderEntry type="library" name="Gradle: com.google.protobuf:protobuf-java:3.4.0" level="project" /> + </component> + <component name="TestModuleProperties" production-module="nexusproto_main" /> +</module> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/.idea/vcs.xml ---------------------------------------------------------------------- diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="VcsDirectoryMappings"> + <mapping directory="$PROJECT_DIR$" vcs="Git" /> + </component> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index dc0bb41..8265730 100644 --- a/build.gradle +++ b/build.gradle @@ -1,35 +1,39 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* http://www.apache.org/licenses/LICENSE-2.0 +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + import java.nio.file.* import java.nio.file.attribute.BasicFileAttributes -buildscript { - repositories { - mavenCentral() - jcenter() - } - dependencies { - classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.3' - } +plugins { + id 'java' + id 'idea' + id 'com.google.protobuf' version '0.8.3' + id 'maven' + id 'maven-publish' + id 'project-report' + id 'net.researchgate.release' version '2.6.0' } repositories { mavenCentral() } - -apply plugin: 'java' -apply plugin: 'com.google.protobuf' -apply plugin: 'maven' -apply plugin: 'maven-publish' -apply plugin: 'project-report' - - -group = 'gov.nasa.jpl.nexus.ningester' -version = '1.0.0.RELEASE' - ext { genDirectory = "$projectDir/gen" distDirectory = "$projectDir/distrobution" - pythonBuildDirPath = "${file(buildDir.path + '/python/ningesterproto').path}" + pythonBuildDirPath = "${file(buildDir.path + '/python/nexusproto').path}" } protobuf { @@ -45,7 +49,7 @@ protobuf { plugins { // Define a plugin with name 'grpc' grpc { - artifact = 'io.grpc:protoc-gen-grpc-java:1.7.0' + artifact = 'io.grpc:protoc-gen-grpc-java:1.9.0' } } @@ -94,16 +98,29 @@ assemble.doLast { } }) - Files.move(file(pythonbuilddir.path + '/ningesterproto/setup.py').toPath(), file(pythonbuilddir.path + '/setup.py').toPath(), StandardCopyOption.REPLACE_EXISTING) + def setupPy = file(pythonbuilddir.path + '/nexusproto/setup.py').text + setupPy = setupPy.replaceAll(/GRADLE_PROJECT_VERSION/, "$project.version") + file(pythonbuilddir.path + '/nexusproto/setup.py').write(setupPy) + + Files.move(file(pythonbuilddir.path + '/nexusproto/setup.py').toPath(), file(pythonbuilddir.path + '/setup.py').toPath(), StandardCopyOption.REPLACE_EXISTING) File generatedPython = file("$genDirectory/main/python/DataTile_pb2.py") - Files.copy(generatedPython.toPath(), file(pythonbuilddir.path + '/ningesterproto/DataTile_pb2.py').toPath(), StandardCopyOption.REPLACE_EXISTING) + Files.copy(generatedPython.toPath(), file(pythonbuilddir.path + '/nexusproto/DataTile_pb2.py').toPath(), StandardCopyOption.REPLACE_EXISTING) +} + +updateVersion.doLast { + File pythonbuilddir = file(pythonBuildDirPath) + + def setupPy = file(pythonbuilddir.path + '/nexusproto/setup.py').text + setupPy = setupPy.replaceAll(/GRADLE_PROJECT_VERSION/, "$project.version") + file(pythonbuilddir.path + '/nexusproto/setup.py').write(setupPy) + } task tarPython(type: Tar, dependsOn: [assemble]) { destinationDir = file("distrobution") - archiveName = 'ningesterproto.tar.gz' + archiveName = 'nexusproto.tar.gz' compression = Compression.GZIP from(file(buildDir.path + '/python')) { include '**/*' http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/gradle.properties ---------------------------------------------------------------------- diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000..bce49ec --- /dev/null +++ b/gradle.properties @@ -0,0 +1,3 @@ +group = 'org.apache.sdap' +version=1.0.1-SNAPSHOT +sourceCompatibility = 1.8 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/gradle/wrapper/gradle-wrapper.jar ---------------------------------------------------------------------- diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index d1645de..610d630 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/gradle/wrapper/gradle-wrapper.properties ---------------------------------------------------------------------- diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 74a41c0..b4965f4 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,4 +1,4 @@ -#Tue Oct 17 11:06:47 PDT 2017 +#Thu Jan 18 14:53:09 PST 2018 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/gradlew.bat ---------------------------------------------------------------------- diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..f955316 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/nexusproto.iml ---------------------------------------------------------------------- diff --git a/nexusproto.iml b/nexusproto.iml new file mode 100644 index 0000000..fd86645 --- /dev/null +++ b/nexusproto.iml @@ -0,0 +1,13 @@ +<?xml version="1.0" encoding="UTF-8"?> +<module external.linked.project.id="nexusproto" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="'org.apache.sdap'" external.system.module.version="1.0.1-SNAPSHOT" type="JAVA_MODULE" version="4"> + <component name="NewModuleRootManager" inherit-compiler-output="true"> + <exclude-output /> + <content url="file://$MODULE_DIR$"> + <excludeFolder url="file://$MODULE_DIR$/.gradle" /> + <excludeFolder url="file://$MODULE_DIR$/build" /> + <excludeFolder url="file://$MODULE_DIR$/out" /> + </content> + <orderEntry type="inheritedJdk" /> + <orderEntry type="sourceFolder" forTests="false" /> + </component> +</module> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/src/main/proto/DataTile.proto ---------------------------------------------------------------------- diff --git a/src/main/proto/DataTile.proto b/src/main/proto/DataTile.proto index b947b11..b3ff403 100644 --- a/src/main/proto/DataTile.proto +++ b/src/main/proto/DataTile.proto @@ -1,40 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + syntax = "proto3"; -package gov.nasa.jpl.nexus.ningester.protobuf; -option java_package = "gov.nasa.jpl.nexus.ningester.protobuf"; +package org.apache.sdap.nexusproto; +option java_package = "org.apache.sdap.nexusproto"; option java_multiple_files = true; -service ProtoTileService { - rpc ToTile(stream TileSpecification) returns (stream Tile) {} - rpc ProcessTile(stream Tile) returns (stream Tile){} +message GridTile{ + + ShapedArray latitude = 1; + ShapedArray longitude = 2; + + int64 time = 3; + + ShapedArray variable_data = 4; + + repeated MetaData meta_data = 5; } -message TileSpecification{ +message SwathTile{ - message Slice { - oneof name_or_position { - string dimension_name = 1; - int32 dimension_position = 2; - } - int64 start = 3; - int64 stop = 4; - int64 step = 5; - } + ShapedArray latitude = 1; + ShapedArray longitude = 2; + ShapedArray time = 3; + + ShapedArray variable_data = 4; + + repeated MetaData meta_data = 5; - repeated Slice slices = 1; } -message Tile{ +message TimeSeriesTile{ + + ShapedArray latitude = 1; + ShapedArray longitude = 2; + + int64 time = 3; + + ShapedArray variable_data = 4; + + repeated MetaData meta_data = 5; + +} + +message MetaData{ + string name = 1; + ShapedArray meta_data = 2; +} + +message ShapedArray{ + repeated int32 shape = 1 [packed=true]; + string dtype = 2; + bytes array_data = 3; +} + +message Attribute{ + string name = 1; + repeated string values = 2; +} + +message TileSummary{ string tile_id = 1; - TileSpecification section_spec = 2; + string section_spec = 2; string dataset_name = 3; string granule = 4; string dataset_uuid = 5; - string variable_name = 6; - repeated string metadata_names = 7; + string data_var_name = 6; + repeated Attribute global_attributes = 7; message BBox{ float lat_min = 1; @@ -61,6 +109,19 @@ message Tile{ float kurtosis = 12; } DataStats stats = 9; +} + +message NexusTile{ + TileSummary summary = 1; + TileData tile = 2; +} - bytes data = 10; +message TileData{ + string tile_id = 1; + + oneof tile_type{ + GridTile grid_tile = 2; + SwathTile swath_tile = 3; + TimeSeriesTile time_series_tile = 4; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/src/main/python/nexusproto/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/nexusproto/__init__.py b/src/main/python/nexusproto/__init__.py new file mode 100644 index 0000000..b0c565d --- /dev/null +++ b/src/main/python/nexusproto/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nexusproto.serialization import * http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/src/main/python/nexusproto/serialization.py ---------------------------------------------------------------------- diff --git a/src/main/python/nexusproto/serialization.py b/src/main/python/nexusproto/serialization.py new file mode 100644 index 0000000..a9ba949 --- /dev/null +++ b/src/main/python/nexusproto/serialization.py @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +try: + from StringIO import StringIO +except ImportError: + from io import BytesIO as StringIO + +from nexusproto import DataTile_pb2 as nexusproto +import numpy + + +def from_shaped_array(shaped_array): + memfile = StringIO() + memfile.write(shaped_array.array_data) + memfile.seek(0) + data_array = numpy.load(memfile) + memfile.close() + + return data_array + + +def to_shaped_array(data_array): + shaped_array = nexusproto.ShapedArray() + + shaped_array.shape.extend([dimension_size for dimension_size in data_array.shape]) + shaped_array.dtype = str(data_array.dtype) + + memfile = StringIO() + numpy.save(memfile, data_array) + shaped_array.array_data = memfile.getvalue() + memfile.close() + + return shaped_array + +def to_metadata(name, data_array): + metadata = nexusproto.MetaData() + metadata.name = name + metadata.meta_data.CopyFrom(to_shaped_array(data_array)) + + return metadata http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/src/main/python/nexusproto/setup.py ---------------------------------------------------------------------- diff --git a/src/main/python/nexusproto/setup.py b/src/main/python/nexusproto/setup.py new file mode 100644 index 0000000..c4f0542 --- /dev/null +++ b/src/main/python/nexusproto/setup.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from setuptools import setup + +__version__ = 'GRADLE_PROJECT_VERSION' + +setup( + name='nexusproto', + version=__version__, + url="https://github.com/apache/incubator-sdap-nexusproto";, + + author="Frank Greguska", + + description="Protobufs used while ingesting NEXUS tiles.", + + packages=['nexusproto'], + platforms='any', + + install_requires=[ + 'protobuf' + ], + + classifiers=[ + 'Development Status :: 1 - Pre-Alpha', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3.5', + ] +) http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/src/main/python/ningesterproto/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/ningesterproto/__init__.py b/src/main/python/ningesterproto/__init__.py deleted file mode 100644 index f7fad5c..0000000 --- a/src/main/python/ningesterproto/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -Copyright (c) 2017 Jet Propulsion Laboratory, -California Institute of Technology. All rights reserved -""" - -from serialization import * http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/src/main/python/ningesterproto/serialization.py ---------------------------------------------------------------------- diff --git a/src/main/python/ningesterproto/serialization.py b/src/main/python/ningesterproto/serialization.py deleted file mode 100644 index 64fe207..0000000 --- a/src/main/python/ningesterproto/serialization.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Copyright (c) 2016 Jet Propulsion Laboratory, -California Institute of Technology. All rights reserved -""" -import StringIO - -import numpy - -import nexusproto.NexusContent_pb2 as nexusproto - - -def from_shaped_array(shaped_array): - memfile = StringIO.StringIO() - memfile.write(shaped_array.array_data) - memfile.seek(0) - data_array = numpy.load(memfile) - memfile.close() - - return data_array - - -def to_shaped_array(data_array): - shaped_array = nexusproto.ShapedArray() - - shaped_array.shape.extend([dimension_size for dimension_size in data_array.shape]) - shaped_array.dtype = str(data_array.dtype) - - memfile = StringIO.StringIO() - numpy.save(memfile, data_array) - shaped_array.array_data = memfile.getvalue() - memfile.close() - - return shaped_array - -def to_metadata(name, data_array): - metadata = nexusproto.MetaData() - metadata.name = name - metadata.meta_data.CopyFrom(to_shaped_array(data_array)) - - return metadata http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexusproto/blob/135cf8a0/src/main/python/ningesterproto/setup.py ---------------------------------------------------------------------- diff --git a/src/main/python/ningesterproto/setup.py b/src/main/python/ningesterproto/setup.py deleted file mode 100644 index b1bcaec..0000000 --- a/src/main/python/ningesterproto/setup.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Copyright (c) 2016 Jet Propulsion Laboratory, -California Institute of Technology. All rights reserved -""" -from setuptools import setup - -__version__ = '0.1' - -setup( - name='ningesterproto', - version=__version__, - url="https://github.com/aist-oceanworks";, - - author="Team Nexus", - - description="Protobufs used while ingesting NEXUS tiles.", - - packages=['ningesterproto'], - platforms='any', - - install_requires=[ - 'protobuf' - ], - - classifiers=[ - 'Development Status :: 1 - Pre-Alpha', - 'Intended Audience :: Developers', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2.7', - ] -)
