This is an automated email from the ASF dual-hosted git repository.
nadavha pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git.
from d722228 Merge pull request #43 from anonymousGiraffe/fix-debug-test
new 812ebc4 started migrating the SDK to kotlin
new 382a58a moved the default action resource files to getActionResources
new ab02a64 moving from spark shell to spark jar implementation
new f7ec145 added artifact support to ActionData and
SparkSubmitScalaRunnerProvider
new 435eaef added support for binary actions
new 51587ea fixed the command executed by the jar executor
new 7b78833 spark jars are executing correctly but spark is not loaded to
the container
new 0ffb969 spark is now loaded to the container
new 7a36e3b hybrid executor mode for mesos
new da97347 TestNotifier fixed
new fa9b743 hybrid mode completed
new b71845e switched to the correct feature branch
new b1c1ea2 removed all hardcoded spark version
new 7023909 removed debug messages
new f88be0a Merge pull request #42 from roadan/AMATERASU-8
The 572 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
common/build.gradle | 9 +-
.../common/configuration/enums/ActionStatus.kt | 12 +-
.../amaterasu/common/dataobjects/ActionData.kt | 11 +-
.../amaterasu/common/dataobjects/Artifact.kt} | 2 +-
.../apache/amaterasu/common/dataobjects/Repo.kt} | 2 +-
.../common/execution/actions/Notification.kt} | 8 +-
.../common/execution/actions/Notifier.kt} | 15 +-
.../actions/enums/NotificationLevel.kt} | 10 +-
.../execution/actions/enums/NotificationType.kt} | 8 +-
.../apache/amaterasu/common/utils/ArtifactUtil.kt | 56 +++++
.../common/configuration/ClusterConfig.scala | 4 +-
.../common/dataobjects/ActionDataHelper.scala | 32 ---
.../common/execution/actions/Notifier.scala | 56 -----
.../executor/common/executors/ActiveNotifier.scala | 17 +-
.../mesos/executors/MesosActionsExecutor.scala | 4 +-
.../executor/mesos/executors/MesosNotifier.scala | 17 +-
.../executor/yarn/executors/YarnNotifier.scala | 6 +-
.../spark/dispatcher/SparkSetupProvider.scala | 4 +-
.../runners/providers/PySparkRunnerProvider.scala | 4 +-
.../providers/SparkScalaRunnerProvider.scala | 5 +-
.../providers/SparkShellScalaRunnerProvider.scala | 30 ---
.../providers/SparkSubmitScalaRunnerProvider.scala | 48 ++++
frameworks/spark/runner/build.gradle | 3 -
.../spark/runner/pyspark/PySparkEntryPoint.java | 2 +-
.../spark/runner/pyspark/PySparkRunner.scala | 2 +-
.../spark/runner/repl/SparkRunnerHelper.scala | 4 +-
.../spark/runner/repl/SparkScalaRunner.scala | 8 +-
.../spark/runner/sparksql/SparkSqlRunner.scala | 2 +-
.../runner/src/test/resources/amaterasu.properties | 14 ++
.../apache/amaterasu/utilities/TestNotifier.scala | 6 +-
leader-common/build.gradle | 7 +-
.../amaterasu/leader/common/dsl/JobParser.kt | 40 +++-
.../leader/common/execution/JobManager.kt | 33 +--
.../leader/common/execution/actions/Action.kt | 22 +-
.../leader/common/execution/actions/ErrorAction.kt | 4 +-
.../common/execution/actions/SequentialAction.kt | 4 +-
.../execution/actions/SequentialActionBase.kt | 12 +-
.../common/utilities/ActiveReportListener.kt | 64 ++++++
.../leader/common/utilities/MessagingClientUtil.kt | 57 +++++
.../common/utilities/ActiveReportListener.scala | 70 ------
.../leader/common/utilities/DataLoader.scala | 7 +-
.../common/configuration/ConfigManagerTests.kt | 1 -
.../leader/common/dsl/JobParserArtifactTests.kt | 69 ++++++
.../resources/{maki.yml => artifact-maki.yaml} | 37 +--
leader/build.gradle | 14 +-
.../scala/org/apache/amaterasu/leader/Kami.scala | 111 ---------
.../org/apache/amaterasu/leader/dsl/GitUtil.scala | 45 ----
.../amaterasu/leader/execution/JobLoader.scala | 4 +-
.../apache/amaterasu/leader/mesos/Launcher.scala | 63 -----
.../mesos/schedulers/AmaterasuScheduler.scala | 2 +-
.../leader/mesos/schedulers/ClusterScheduler.scala | 135 -----------
.../leader/mesos/schedulers/JobScheduler.scala | 254 +++++++++++----------
.../amaterasu/leader/utilities/HttpServer.scala | 4 +-
.../amaterasu/leader/yarn/ApplicationMaster.scala | 64 +++---
.../leader/yarn/YarnNMCallbackHandler.scala | 2 +-
.../leader/yarn/YarnRMCallbackHandler.scala | 6 +-
leader/src/test/resources/simple-maki.yml | 3 +-
.../common/execution/ActionStatusTests.scala | 6 +-
.../common/execution/JobExecutionTests.scala | 48 ++--
.../common/execution/JobRestoreTests.scala | 12 +-
.../leader/mesos/ClusterSchedulerTests.scala | 58 -----
project/build.properties | 1 +
sdk/build.gradle | 68 +++++-
.../sdk/frameworks/RunnerSetupProvider.java | 31 ---
.../configuration/DriverConfiguration.java | 37 ---
.../sdk/frameworks/FrameworkSetupProvider.kt} | 25 +-
.../sdk/frameworks/RunnerSetupProvider.kt | 39 ++++
.../configuration/DriverConfiguration.kt | 6 +-
.../sdk/frameworks/RunnerSetupProviderTests.kt | 22 ++
.../amaterasu/sdk/frameworks/TestRunnerProvider.kt | 23 ++
70 files changed, 897 insertions(+), 1014 deletions(-)
copy
common/src/main/{scala/org/apache/amaterasu/common/dataobjects/Export.scala =>
kotlin/org/apache/amaterasu/common/dataobjects/Artifact.kt} (90%)
copy
common/src/main/{scala/org/apache/amaterasu/common/dataobjects/Export.scala =>
kotlin/org/apache/amaterasu/common/dataobjects/Repo.kt} (92%)
copy
common/src/main/{scala/org/apache/amaterasu/common/dataobjects/ExecData.scala
=> kotlin/org/apache/amaterasu/common/execution/actions/Notification.kt} (70%)
copy
common/src/main/{scala/org/apache/amaterasu/common/runtime/Environment.scala =>
kotlin/org/apache/amaterasu/common/execution/actions/Notifier.kt} (74%)
mode change 100755 => 100644
copy common/src/main/kotlin/org/apache/amaterasu/common/{logging/KLogging.kt
=> execution/actions/enums/NotificationLevel.kt} (82%)
copy
common/src/main/{scala/org/apache/amaterasu/common/dataobjects/Export.scala =>
kotlin/org/apache/amaterasu/common/execution/actions/enums/NotificationType.kt}
(82%)
create mode 100644
common/src/main/kotlin/org/apache/amaterasu/common/utils/ArtifactUtil.kt
delete mode 100644
common/src/main/scala/org/apache/amaterasu/common/dataobjects/ActionDataHelper.scala
delete mode 100755
common/src/main/scala/org/apache/amaterasu/common/execution/actions/Notifier.scala
delete mode 100644
frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/SparkShellScalaRunnerProvider.scala
create mode 100644
frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/SparkSubmitScalaRunnerProvider.scala
create mode 100644
leader-common/src/main/kotlin/org/apache/amaterasu/leader/common/utilities/ActiveReportListener.kt
create mode 100644
leader-common/src/main/kotlin/org/apache/amaterasu/leader/common/utilities/MessagingClientUtil.kt
delete mode 100644
leader-common/src/main/scala/org/apache/amaterasu/leader/common/utilities/ActiveReportListener.scala
create mode 100644
leader-common/src/test/kotlin/org/apache/amaterasu/leader/common/dsl/JobParserArtifactTests.kt
copy leader-common/src/test/resources/{maki.yml => artifact-maki.yaml} (64%)
delete mode 100755 leader/src/main/scala/org/apache/amaterasu/leader/Kami.scala
delete mode 100755
leader/src/main/scala/org/apache/amaterasu/leader/dsl/GitUtil.scala
delete mode 100755
leader/src/main/scala/org/apache/amaterasu/leader/mesos/Launcher.scala
delete mode 100755
leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/ClusterScheduler.scala
delete mode 100755
leader/src/test/scala/org/apache/amaterasu/leader/mesos/ClusterSchedulerTests.scala
create mode 100644 project/build.properties
delete mode 100644
sdk/src/main/java/org/apache/amaterasu/sdk/frameworks/RunnerSetupProvider.java
delete mode 100644
sdk/src/main/java/org/apache/amaterasu/sdk/frameworks/configuration/DriverConfiguration.java
rename
sdk/src/main/{java/org/apache/amaterasu/sdk/frameworks/FrameworkSetupProvider.java
=> kotlin/org/apache/amaterasu/sdk/frameworks/FrameworkSetupProvider.kt} (62%)
create mode 100644
sdk/src/main/kotlin/org/apache/amaterasu/sdk/frameworks/RunnerSetupProvider.kt
rename leader/src/main/scala/org/apache/amaterasu/leader/package.scala =>
sdk/src/main/kotlin/org/apache/amaterasu/sdk/frameworks/configuration/DriverConfiguration.kt
(86%)
mode change 100755 => 100644
create mode 100644
sdk/src/test/kotlin/org/apache/amaterasu/sdk/frameworks/RunnerSetupProviderTests.kt
create mode 100644
sdk/src/test/kotlin/org/apache/amaterasu/sdk/frameworks/TestRunnerProvider.kt
