This is an automated email from the ASF dual-hosted git repository. yaniv pushed a commit to branch version-0.2.0-incubating-rc4 in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git
commit abe3f2999abfdc3eb002a97fca97b89b139217e5 Merge: 11f0b30 1273740 Author: Yaniv Rodenski <[email protected]> AuthorDate: Sun Jul 1 10:02:42 2018 +1000 Merge pull request #27 from arunma/RC4-RefactorFramework AMATERASU-24 Refactor Spark out of Amaterasu executor to it's own project build.gradle | 4 - .../common/execution/actions/Notifier.scala | 4 +- executor/build.gradle | 14 +--- .../mesos/executors/MesosActionsExecutor.scala | 4 +- .../executor/yarn/executors/ActionsExecutor.scala | 8 +- {executor => frameworks/spark/runner}/build.gradle | 14 ++-- .../spark/runner/pyspark}/PySparkEntryPoint.java | 11 +-- .../spark/runner/src/main}/resources/codegen.py | 0 .../spark/runner/src/main}/resources/runtime.py | 0 .../main/resources/spark-version-info.properties | 11 +++ .../spark/runner}/src/main/resources/spark_intp.py | 0 .../spark/runner}/SparkRunnersProvider.scala | 10 +-- .../runner/pyspark}/PySparkExecutionQueue.scala | 2 +- .../spark/runner/pyspark}/PySparkResultQueue.scala | 4 +- .../spark/runner/pyspark}/PySparkRunner.scala | 10 +-- .../spark/runner/pyspark}/ResultQueue.scala | 2 +- .../spark/runner/repl}/AmaSparkILoop.scala | 2 +- .../spark/runner/repl}/SparkRunnerHelper.scala | 14 ++-- .../spark/runner/repl}/SparkScalaRunner.scala | 6 +- .../spark/runner/sparkr}/SparkRRunner.scala | 28 +++---- .../spark/runner/sparksql}/SparkSqlRunner.scala | 9 ++- .../resources/SparkSql/csv/SparkSqlTestCsv.csv | 0 .../resources/SparkSql/json/SparkSqlTestData.json | 0 ...548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc | Bin ...548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc | Bin .../src/test/resources/SparkSql/parquet/_SUCCESS | 0 .../resources/SparkSql/parquet/_common_metadata | Bin .../src/test/resources/SparkSql/parquet/_metadata | Bin ...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin ...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin .../src/test/resources/amaterasu.properties | 0 .../spark/runner/src/test}/resources/codegen.py | 0 .../runner}/src/test/resources/py4j-0.10.4-src.zip | Bin .../spark/runner}/src/test/resources/py4j.tar.gz | Bin .../src/test/resources/pyspark-with-amacontext.py | 0 .../runner}/src/test/resources/pyspark.tar.gz | Bin .../spark/runner}/src/test/resources/pyspark.zip | Bin .../spark/runner/src/test}/resources/runtime.py | 0 .../runner}/src/test/resources/simple-pyspark.py | 0 .../src/test/resources/simple-python-err.py | 0 .../runner}/src/test/resources/simple-python.py | 0 .../runner}/src/test/resources/simple-spark.scala | 5 +- .../test}/resources/spark-version-info.properties | 0 .../spark/runner}/src/test/resources/spark_intp.py | 0 .../spark/runner}/src/test/resources/step-2.scala | 3 +- ...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin ...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin .../framework}/spark/SparkTestsSuite.scala | 10 +-- .../spark/runner}/RunnersLoadingTests.scala | 2 +- .../spark/runner/pyspark}/PySparkRunnerTests.scala | 3 +- .../spark/runner/repl}/SparkScalaRunnerTests.scala | 9 +-- .../runner/sparksql}/SparkSqlRunnerTests.scala | 9 +-- .../apache/amaterasu/utilities/TestNotifier.scala | 0 frameworks/spark/runtime/build.gradle | 89 +++++++++++++++++++++ .../framework/spark}/runtime/AmaContext.scala | 8 +- gradle/wrapper/gradle-wrapper.properties | 4 +- gradlew | 16 ---- gradlew.bat | 18 ----- .../apache/amaterasu/leader/yarn/ArgsParser.java | 6 +- .../org/apache/amaterasu/leader/yarn/Client.java | 23 +++++- .../apache/amaterasu/leader/dsl/JobParser.scala | 4 +- .../frameworks/spark/SparkSetupProvider.scala | 1 - .../mesos/schedulers/AmaterasuScheduler.scala | 1 - .../leader/mesos/schedulers/JobScheduler.scala | 5 +- .../leader/utilities/ActiveReportListener.scala | 2 - .../amaterasu/leader/utilities/HttpServer.scala | 7 +- .../amaterasu/leader/yarn/ApplicationMaster.scala | 39 ++++++--- .../leader/yarn/YarnRMCallbackHandler.scala | 5 +- leader/src/main/scripts/ama-start-mesos.sh | 4 +- leader/src/main/scripts/ama-start-yarn.sh | 4 +- .../leader/mesos/ClusterSchedulerTests.scala | 2 +- .../amaterasu/utilities/HttpServerTests.scala | 6 -- settings.gradle | 14 +++- 73 files changed, 256 insertions(+), 200 deletions(-)
