This is an automated email from the ASF dual-hosted git repository. yaniv pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git
commit 8b07446a8619365f398759b1099320d8fc2cbc3f Merge: f9306bc 55d4d5a Author: Yaniv Rodenski <[email protected]> AuthorDate: Thu Jan 11 19:45:35 2018 +1100 Merge pull request #3 from eyalbenivri/feature/yarn-support-pyfix Feature/yarn support pyfix common/build.gradle | 15 +- .../common/configuration/ClusterConfig.scala | 102 ++- .../common/configuration/SparkConfig.scala | 0 .../common/configuration/enums/ActionStatus.scala | 2 +- .../common/dataobjects/ActionData.scala | 4 +- .../common/dataobjects/ActionDataHelper.scala | 14 + .../common/dataobjects/ExecData.scala | 0 .../common/dataobjects/Export.scala | 0 .../common/dataobjects/JobData.scala | 0 .../common/dataobjects/TaskData.scala | 0 .../common/execution/actions/Notifier.scala | 3 - .../execution/dependencies/Dependencies.scala | 3 - .../common/logging/Logging.scala | 0 .../common/runtime/Environment.scala | 3 - executor/build.gradle | 16 +- .../executors/ProvidersFactory.scala | 7 +- .../spark/PySpark/PySparkExecutionQueue.scala | 6 +- .../runners/spark/PySpark/PySparkResultQueue.scala | 3 - .../runners/spark/PySpark/PySparkRunner.scala | 11 +- .../runners/spark/PySpark/ResultQueue.scala | 6 +- .../actions/runners/spark/SparkRRunner.scala | 4 +- .../runners/spark/SparkRunnersProvider.scala | 29 +- .../runners/spark/SparkSql/SparkSqlRunner.scala | 1 - .../executor/mesos/executors/ActionsExecutor.scala | 11 +- .../executor/mesos/executors/MesosNotifier.scala | 4 +- .../executor/yarn/executors/ActionsExecutor.scala | 94 +++ .../executor/yarn/executors/YarnNotifier.scala | 23 + .../spark/repl/amaterasu/AmaSparkILoop.scala | 2 +- .../runners/spark/SparkRunnerHelper.scala | 79 ++- .../amaterasu/runners/spark/SparkScalaRunner.scala | 24 +- executor/src/test/resources/amaterasu.properties | 10 +- executor/src/test/resources/py4j/backport.py | 233 ------- executor/src/test/resources/py4j/clientserver.py | 550 --------------- executor/src/test/resources/py4j/signals.py | 123 ---- .../src/test/resources/py4j/tests/benchmark1.py | 146 ---- .../resources/py4j/tests/client_server_test.py | 537 --------------- .../src/test/resources/py4j/tests/instrumented.py | 166 ----- .../src/test/resources/py4j/tests/java_tls_test.py | 100 --- .../test/resources/py4j/tests/memory_leak_test.py | 767 --------------------- .../py4j/tests/py4j_callback_listener_example.py | 25 - .../py4j/tests/py4j_callback_recursive_example.py | 78 --- .../test/resources/py4j/tests/py4j_signals_test.py | 151 ---- .../src/test/resources/py4j/tests/signals_test.py | 97 --- .../src/test/resources/pyspark-with-amacontext.py | 5 +- executor/src/test/resources/pyspark.zip | Bin 442927 -> 454461 bytes executor/src/test/resources/runtime.py | 18 +- executor/src/test/resources/simple-pyspark.py | 13 +- executor/src/test/resources/spark_intp.py | 39 +- .../RunnersTests/RunnersLoadingTests.scala | 19 - .../RunnersTests/RunnersLoadingTests.scala | 2 +- .../amaterasu/spark/PySparkRunnerTests.scala | 24 +- .../amaterasu/spark/SparkSqlRunnerTests.scala | 8 +- .../apache/amaterasu/spark/SparkTestsSuite.scala | 19 +- .../apache/amaterasu/utilities/TestNotifier.scala | 4 +- leader/build.gradle | 11 +- .../apache/amaterasu/leader/yarn/ArgsParser.java | 61 ++ .../org/apache/amaterasu/leader/yarn/Client.java | 268 +++++++ .../org/apache/amaterasu/leader/yarn/JobOpts.java | 12 + leader/src/main/resources/log4j.properties | 8 + .../amaterasu/leader/execution/JobLoader.scala | 6 +- .../amaterasu/leader/execution/JobManager.scala | 6 +- .../leader/execution/actions/Action.scala | 2 +- .../execution/actions/SequentialAction.scala | 6 +- .../amaterasu/leader/mesos/JobLauncher.scala | 113 --- .../amaterasu/leader/mesos/MesosJobLauncher.scala | 45 ++ .../leader/mesos/schedulers/JobScheduler.scala | 19 +- .../apache/amaterasu/leader/utilities/Args.scala | 67 ++ .../leader/utilities/BaseJobLauncher.scala | 26 + .../amaterasu/leader/utilities/DataLoader.scala | 29 +- .../amaterasu/leader/utilities/HttpServer.scala | 2 +- .../amaterasu/leader/yarn/ApplicationMaster.scala | 387 +++++++++++ .../leader/yarn/YarnNMCallbackHandler.scala | 53 ++ .../leader/yarn/YarnRMCallbackHandler.scala | 135 ++++ .../scripts/{ama-start.sh => ama-start-mesos.sh} | 5 + .../scripts/{ama-start.sh => ama-start-yarn.sh} | 46 +- leader/src/main/scripts/amaterasu.properties | 18 +- leader/src/main/scripts/log4j.properties | 8 + leader/src/test/resources/amaterasu.properties | 1 + .../amaterasu/common/execution/ActionTests.scala | 7 +- .../common/execution/JobRestoreTests.scala | 8 +- .../amaterasu/leader/mesos/MesosTestUtil.scala | 4 +- .../amaterasu/utilities/HttpServerTests.scala | 31 +- .../org/apache/amaterasu/sdk/AmaterasuRunner.java | 3 - .../org/apache/amaterasu/sdk/RunnersProvider.java | 5 +- 84 files changed, 1614 insertions(+), 3378 deletions(-) -- To stop receiving notification emails like this one, please contact "[email protected]" <[email protected]>.
