This is an automated email from the ASF dual-hosted git repository.
yaniv pushed a change to branch version-0.2.0-incubating-rc4
in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git.
from abe3f29 Merge pull request #27 from arunma/RC4-RefactorFramework
add 4e21ef8 Revert "AMATERASU-24 Refactor Spark out of Amaterasu executor
to it's own project"
new 5c66d9d Merge pull request #29 from
apache/revert-27-RC4-RefactorFramework
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build.gradle | 4 +
.../common/execution/actions/Notifier.scala | 4 +-
executor/build.gradle | 14 ++-
.../runners/spark/PySpark}/PySparkEntryPoint.java | 11 +-
.../src/main}/resources/codegen.py | 0
.../src/main}/resources/runtime.py | 0
.../main}/resources/spark-version-info.properties | 0
.../src/main/resources/spark_intp.py | 0
.../spark/PySpark}/PySparkExecutionQueue.scala | 2 +-
.../spark/PySpark}/PySparkResultQueue.scala | 4 +-
.../runners/spark/PySpark}/PySparkRunner.scala | 10 +-
.../runners/spark/PySpark}/ResultQueue.scala | 2 +-
.../actions/runners/spark}/SparkRRunner.scala | 28 ++---
.../runners/spark}/SparkRunnersProvider.scala | 10 +-
.../runners/spark/SparkSql}/SparkSqlRunner.scala | 9 +-
.../mesos/executors/MesosActionsExecutor.scala | 4 +-
.../amaterasu/executor}/runtime/AmaContext.scala | 8 +-
.../executor/yarn/executors/ActionsExecutor.scala | 8 +-
.../spark/repl/amaterasu}/AmaSparkILoop.scala | 2 +-
.../runners/spark}/SparkRunnerHelper.scala | 14 +--
.../runners/spark}/SparkScalaRunner.scala | 6 +-
.../resources/SparkSql/csv/SparkSqlTestCsv.csv | 0
.../resources/SparkSql/json/SparkSqlTestData.json | 0
...548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc | Bin
...548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc | Bin
.../src/test/resources/SparkSql/parquet/_SUCCESS | 0
.../resources/SparkSql/parquet/_common_metadata | Bin
.../src/test/resources/SparkSql/parquet/_metadata | Bin
...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
.../src/test/resources/amaterasu.properties | 0
.../src/test}/resources/codegen.py | 0
.../src/test/resources/py4j-0.10.4-src.zip | Bin
.../src/test/resources/py4j.tar.gz | Bin
.../src/test/resources/pyspark-with-amacontext.py | 0
.../src/test/resources/pyspark.tar.gz | Bin
.../src/test/resources/pyspark.zip | Bin
.../src/test}/resources/runtime.py | 0
.../src/test/resources/simple-pyspark.py | 0
.../src/test/resources/simple-python-err.py | 0
.../src/test/resources/simple-python.py | 0
.../src/test/resources/simple-spark.scala | 5 +-
.../src/test/resources/spark_intp.py | 0
.../src/test/resources/step-2.scala | 3 +-
...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
.../RunnersTests}/RunnersLoadingTests.scala | 2 +-
.../amaterasu/spark}/PySparkRunnerTests.scala | 3 +-
.../amaterasu/spark}/SparkScalaRunnerTests.scala | 9 +-
.../amaterasu/spark}/SparkSqlRunnerTests.scala | 9 +-
.../apache/amaterasu}/spark/SparkTestsSuite.scala | 10 +-
.../apache/amaterasu/utilities/TestNotifier.scala | 0
frameworks/spark/runner/build.gradle | 118 ---------------------
.../main/resources/spark-version-info.properties | 11 --
frameworks/spark/runtime/build.gradle | 89 ----------------
gradle/wrapper/gradle-wrapper.properties | 4 +-
gradlew | 16 +++
gradlew.bat | 18 ++++
.../apache/amaterasu/leader/yarn/ArgsParser.java | 6 +-
.../org/apache/amaterasu/leader/yarn/Client.java | 23 +---
.../apache/amaterasu/leader/dsl/JobParser.scala | 4 +-
.../frameworks/spark/SparkSetupProvider.scala | 1 +
.../mesos/schedulers/AmaterasuScheduler.scala | 1 +
.../leader/mesos/schedulers/JobScheduler.scala | 5 +-
.../leader/utilities/ActiveReportListener.scala | 2 +
.../amaterasu/leader/utilities/HttpServer.scala | 7 +-
.../amaterasu/leader/yarn/ApplicationMaster.scala | 39 +++----
.../leader/yarn/YarnRMCallbackHandler.scala | 5 +-
leader/src/main/scripts/ama-start-mesos.sh | 4 +-
leader/src/main/scripts/ama-start-yarn.sh | 4 +-
.../leader/mesos/ClusterSchedulerTests.scala | 2 +-
.../amaterasu/utilities/HttpServerTests.scala | 6 ++
settings.gradle | 14 +--
73 files changed, 192 insertions(+), 368 deletions(-)
rename
{frameworks/spark/runner/src/main/java/org/apache/amaterasu/framework/spark/runner/pyspark
=>
executor/src/main/java/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark}/PySparkEntryPoint.java
(92%)
rename {frameworks/spark/runner/src/test =>
executor/src/main}/resources/codegen.py (100%)
rename {frameworks/spark/runner/src/test =>
executor/src/main}/resources/runtime.py (100%)
rename {frameworks/spark/runner/src/test =>
executor/src/main}/resources/spark-version-info.properties (100%)
rename {frameworks/spark/runner => executor}/src/main/resources/spark_intp.py
(100%)
rename
{frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark
=>
executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark}/PySparkExecutionQueue.scala
(94%)
rename
{frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark
=>
executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark}/PySparkResultQueue.scala
(85%)
rename
{frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark
=>
executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark}/PySparkRunner.scala
(95%)
mode change 100644 => 100755
rename
{frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark
=>
executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark}/ResultQueue.scala
(94%)
rename
{frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparkr
=>
executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark}/SparkRRunner.scala
(69%)
rename
{frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner
=>
executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark}/SparkRunnersProvider.scala
(93%)
rename
{frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparksql
=>
executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkSql}/SparkSqlRunner.scala
(96%)
rename
{frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/framework/spark
=>
executor/src/main/scala/org/apache/amaterasu/executor}/runtime/AmaContext.scala
(93%)
mode change 100644 => 100755
rename
{frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl
=>
executor/src/main/scala/org/apache/spark/repl/amaterasu}/AmaSparkILoop.scala
(95%)
rename
{frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl
=>
executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark}/SparkRunnerHelper.scala
(94%)
rename
{frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl
=>
executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark}/SparkScalaRunner.scala
(97%)
rename {frameworks/spark/runner =>
executor}/src/test/resources/SparkSql/csv/SparkSqlTestCsv.csv (100%)
rename {frameworks/spark/runner =>
executor}/src/test/resources/SparkSql/json/SparkSqlTestData.json (100%)
rename {frameworks/spark/runner =>
executor}/src/test/resources/SparkSql/parquet/.part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc
(100%)
rename {frameworks/spark/runner =>
executor}/src/test/resources/SparkSql/parquet/.part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc
(100%)
rename {frameworks/spark/runner =>
executor}/src/test/resources/SparkSql/parquet/_SUCCESS (100%)
rename {frameworks/spark/runner =>
executor}/src/test/resources/SparkSql/parquet/_common_metadata (100%)
rename {frameworks/spark/runner =>
executor}/src/test/resources/SparkSql/parquet/_metadata (100%)
rename {frameworks/spark/runner/src/test/resources/tmp/job/start/x =>
executor/src/test/resources/SparkSql/parquet}/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
(100%)
rename {frameworks/spark/runner/src/test/resources/tmp/job/start/x =>
executor/src/test/resources/SparkSql/parquet}/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
(100%)
rename {frameworks/spark/runner =>
executor}/src/test/resources/amaterasu.properties (100%)
rename {frameworks/spark/runner/src/main =>
executor/src/test}/resources/codegen.py (100%)
rename {frameworks/spark/runner =>
executor}/src/test/resources/py4j-0.10.4-src.zip (100%)
rename {frameworks/spark/runner => executor}/src/test/resources/py4j.tar.gz
(100%)
rename {frameworks/spark/runner =>
executor}/src/test/resources/pyspark-with-amacontext.py (100%)
rename {frameworks/spark/runner => executor}/src/test/resources/pyspark.tar.gz
(100%)
rename {frameworks/spark/runner => executor}/src/test/resources/pyspark.zip
(100%)
rename {frameworks/spark/runner/src/main =>
executor/src/test}/resources/runtime.py (100%)
rename {frameworks/spark/runner =>
executor}/src/test/resources/simple-pyspark.py (100%)
rename {frameworks/spark/runner =>
executor}/src/test/resources/simple-python-err.py (100%)
rename {frameworks/spark/runner =>
executor}/src/test/resources/simple-python.py (100%)
rename {frameworks/spark/runner =>
executor}/src/test/resources/simple-spark.scala (83%)
rename {frameworks/spark/runner => executor}/src/test/resources/spark_intp.py
(100%)
rename {frameworks/spark/runner => executor}/src/test/resources/step-2.scala
(94%)
rename {frameworks/spark/runner/src/test/resources/SparkSql/parquet =>
executor/src/test/resources/tmp/job/start/x}/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
(100%)
rename {frameworks/spark/runner/src/test/resources/SparkSql/parquet =>
executor/src/test/resources/tmp/job/start/x}/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
(100%)
rename
{frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner
=>
executor/src/test/scala/org/apache/amaterasu/RunnersTests}/RunnersLoadingTests.scala
(96%)
rename
{frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/pyspark
=>
executor/src/test/scala/org/apache/amaterasu/spark}/PySparkRunnerTests.scala
(96%)
rename
{frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/repl
=>
executor/src/test/scala/org/apache/amaterasu/spark}/SparkScalaRunnerTests.scala
(92%)
rename
{frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/sparksql
=>
executor/src/test/scala/org/apache/amaterasu/spark}/SparkSqlRunnerTests.scala
(96%)
rename {frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework
=> executor/src/test/scala/org/apache/amaterasu}/spark/SparkTestsSuite.scala
(90%)
rename {frameworks/spark/runner =>
executor}/src/test/scala/org/apache/amaterasu/utilities/TestNotifier.scala
(100%)
delete mode 100644 frameworks/spark/runner/build.gradle
delete mode 100644
frameworks/spark/runner/src/main/resources/spark-version-info.properties
delete mode 100644 frameworks/spark/runtime/build.gradle
