This is an automated email from the ASF dual-hosted git repository. yaniv pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git
commit 4f48c704f8284b3e1c983184e1fd014ec9745f5b Merge: 6992ebf f8b8fc4 Author: Yaniv Rodenski <[email protected]> AuthorDate: Mon May 6 12:36:59 2019 +1000 Merge branch 'nadav-har-tzvi-AMATERASU-45-Rebase' common/build.gradle | 20 ++- .../execution/dependencies/PythonDependencies.kt | 2 +- .../org/apache/amaterasu/common/utils/FileUtil.kt | 100 +++++++++++ .../common/configuration/ClusterConfig.scala | 33 +++- .../common/configuration/SparkConfig.scala | 2 +- .../apache/amaterasu/common/utils/FileTestUtils.kt | 25 +++ .../python/dispatcher}/build.gradle | 50 +++--- .../python/dispatcher/PythonSetupProvider.kt | 53 ++++++ .../runners/providers/BasicPythonRunnerProvider.kt | 36 ++-- .../runners/providers/PandasRunnerProvider.kt | 20 +++ .../runners/providers/PythonRunnerProviderBase.kt | 76 +++++++++ .../dispatcher/BasicPythonRunnerProviderTests.kt | 88 ++++++++++ frameworks/python/pandas_runtime/__init__.py | 16 ++ .../amaterasu_pandas.egg-info/PKG-INFO | 28 +++ .../amaterasu_pandas.egg-info/SOURCES.txt | 11 ++ .../amaterasu_pandas.egg-info/dependency_links.txt | 1 + .../amaterasu_pandas.egg-info/entry_points.txt | 3 + .../amaterasu_pandas.egg-info/requires.txt | 8 + .../amaterasu_pandas.egg-info/top_level.txt | 2 + frameworks/python/pandas_runtime/build.gradle | 13 ++ .../dist/amaterasu_pandas-0.2.0-incubating-rc4.zip | Bin 0 -> 8304 bytes .../pandas_runtime/pandas_runtime/__init__.py | 16 ++ .../pandas_runtime/pandas_runtime/datasets.py | 134 +++++++++++++++ .../pandas_runtime/pandas_runtime/runtime.py | 35 ++++ frameworks/python/pandas_runtime/setup.py | 51 ++++++ frameworks/python/pandas_runtime/tests/__init__.py | 0 frameworks/python/python_runtime/__init__.py | 16 ++ .../amaterasu_python.egg-info/PKG-INFO | 28 +++ .../amaterasu_python.egg-info/SOURCES.txt | 9 + .../amaterasu_python.egg-info/dependency_links.txt | 1 + .../amaterasu_python.egg-info/entry_points.txt | 3 + .../amaterasu_python.egg-info/requires.txt | 1 + .../amaterasu_python.egg-info/top_level.txt | 1 + frameworks/python/python_runtime/build.gradle | 13 ++ .../dist/amaterasu_python-0.2.0-incubating-rc4.zip | Bin 0 -> 6167 bytes .../python_runtime/python_runtime/__init__.py | 16 ++ .../python_runtime/python_runtime/runtime.py | 33 ++++ frameworks/python/python_runtime/setup.py | 51 ++++++ frameworks/spark/dispatcher/build.gradle | 28 ++- .../spark/dispatcher/SparkSetupProvider.kt | 103 +++++++++++ .../runners/providers/PySparkRunnerProvider.kt | 49 ++++++ .../providers/SparkSubmitScalaRunnerProvider.kt | 44 +++++ .../spark/dispatcher/SparkSetupProvider.scala | 119 ------------- .../runners/providers/PySparkRunnerProvider.scala | 53 ------ .../providers/SparkScalaRunnerProvider.scala | 72 -------- .../providers/SparkSubmitScalaRunnerProvider.scala | 52 ------ frameworks/spark/pyspark_runtime/__init__.py | 16 ++ .../amaterasu_pyspark.egg-info/PKG-INFO | 28 +++ .../amaterasu_pyspark.egg-info/SOURCES.txt | 16 ++ .../dependency_links.txt | 1 + .../amaterasu_pyspark.egg-info/entry_points.txt | 3 + .../amaterasu_pyspark.egg-info/requires.txt | 2 + .../amaterasu_pyspark.egg-info/top_level.txt | 2 + .../pyspark_runtime/amaterasu_pyspark/__init__.py | 16 ++ .../pyspark_runtime/amaterasu_pyspark/datasets.py | 73 ++++++++ .../pyspark_runtime/amaterasu_pyspark/runtime.py | 84 +++++++++ frameworks/spark/pyspark_runtime/build.gradle | 13 ++ .../amaterasu_pyspark-0.2.0-incubating-rc4.zip | Bin 0 -> 14488 bytes frameworks/spark/pyspark_runtime/setup.py | 51 ++++++ frameworks/spark/pyspark_runtime/tests/__init__.py | 16 ++ frameworks/spark/pyspark_runtime/tests/base.py | 51 ++++++ .../spark/pyspark_runtime/tests/datasets.yml | 32 ++++ frameworks/spark/pyspark_runtime/tests/env.yml | 32 ++++ .../pyspark_runtime/tests/integration_tests.py | 83 +++++++++ frameworks/spark/pyspark_runtime/tests/runtime.yml | 20 +++ .../spark/pyspark_runtime/tests/runtime_tests.py | 85 ++++++++++ frameworks/spark/pyspark_runtime/tests/sample.py | 24 +++ .../pyspark_runtime/tests/test_scripts/simple.py | 36 ++++ .../spark/pyspark_runtime/tests/unit_tests.py | 62 +++++++ .../spark/runner/SparkRunnersProvider.scala | 42 ++--- .../spark/runner/pyspark/PySparkRunner.scala | 4 - .../spark/runner/repl/SparkRunnerHelper.scala | 6 +- .../frameworks/spark/runner/SparkTestsSuite.scala | 2 +- leader-common/build.gradle | 8 +- .../apache/amaterasu/leader/common/dsl/GitUtil.kt | 2 + .../leader/common/execution/JobManager.kt | 2 +- .../frameworls/FrameworkProvidersFactory.kt | 1 - .../leader/common/utilities/DataLoader.kt | 29 +++- .../amaterasu/leader/common/dsl/GitUtilTests.kt | 18 ++ .../leader/common/dsl/JobParserArtifactTests.kt | 4 +- leader-yarn/build.gradle | 3 +- .../amaterasu/leader/yarn/ApplicationMaster.kt | 57 +++++-- .../org/apache/amaterasu/leader/yarn/Client.kt | 15 +- .../amaterasu/leader/yarn/YarnNMCallbackHandler.kt | 16 +- leader/build.gradle | 5 +- .../amaterasu/leader/mesos/MesosJobLauncher.scala | 1 - .../leader/mesos/schedulers/JobScheduler.scala | 121 ++++++++----- leader/src/main/scripts/ama-start-mesos.sh | 19 ++- leader/src/main/scripts/ama-start-yarn.sh | 8 +- leader/src/main/scripts/amaterasu.properties | 6 +- leader/src/test/resources/amaterasu.properties | 25 +-- .../common/execution/JobExecutionTests.scala | 17 ++ sdk/build.gradle | 10 +- .../sdk/frameworks/RunnerSetupProvider.kt | 38 ++++- sdk_python/__init__.py | 16 ++ sdk_python/amaterasu/__init__.py | 16 ++ sdk_python/amaterasu/base/__init__.py | 2 + sdk_python/amaterasu/base/datasets.py | 116 +++++++++++++ sdk_python/amaterasu/base/runtime.py | 188 +++++++++++++++++++++ sdk_python/amaterasu/pandas/__init__.py | 0 sdk_python/amaterasu/pandas/datasets.py | 134 +++++++++++++++ sdk_python/amaterasu/pandas/runtime.py | 35 ++++ sdk_python/amaterasu/pyspark/__init__.py | 0 sdk_python/amaterasu/pyspark/datasets.py | 73 ++++++++ sdk_python/amaterasu/pyspark/runtime.py | 91 ++++++++++ sdk_python/amaterasu_sdk.egg-info/PKG-INFO | 28 +++ sdk_python/amaterasu_sdk.egg-info/SOURCES.txt | 16 ++ .../amaterasu_sdk.egg-info/dependency_links.txt | 1 + sdk_python/amaterasu_sdk.egg-info/requires.txt | 3 + sdk_python/amaterasu_sdk.egg-info/top_level.txt | 1 + sdk_python/build.gradle | 13 ++ .../dist/amaterasu-sdk-0.2.0-incubating-rc4.zip | Bin 0 -> 15020 bytes sdk_python/setup.py | 60 +++++++ settings.gradle | 21 +++ 114 files changed, 3028 insertions(+), 506 deletions(-)
