This is an automated email from the ASF dual-hosted git repository.
yaniv pushed a commit to branch version-0.2.0-incubating-rc2
in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git
The following commit(s) were added to refs/heads/version-0.2.0-incubating-rc2
by this push:
new 5b4e8eb fixed merge issue
5b4e8eb is described below
commit 5b4e8eb2bd564af37cbcda929720d75323bb1cfb
Author: Yaniv Rodenski <[email protected]>
AuthorDate: Sun Apr 22 18:56:40 2018 +1000
fixed merge issue
---
.../runners/spark/SparkRunnerHelper.scala | 2 +-
.../leader/mesos/schedulers/JobScheduler.scala | 28 ++++++++++++----------
.../amaterasu/leader/yarn/ApplicationMaster.scala | 2 +-
.../leader/yarn/YarnRMCallbackHandler.scala | 2 +-
leader/src/main/scripts/ama-start-yarn.sh | 2 +-
leader/src/main/scripts/amaterasu.properties | 2 +-
6 files changed, 21 insertions(+), 17 deletions(-)
diff --git
a/executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkRunnerHelper.scala
b/executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkRunnerHelper.scala
index abab8a4..0bf7337 100644
---
a/executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkRunnerHelper.scala
+++
b/executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkRunnerHelper.scala
@@ -145,7 +145,7 @@ object SparkRunnerHelper extends Logging {
case "yarn" =>
conf.set("spark.home", config.spark.home)
// TODO: parameterize those
- .setJars(s"executor-${config.version}-all.jar" +: jars)
+ .setJars(s"executor.jar" +: jars)
.set("spark.history.kerberos.keytab",
"/etc/security/keytabs/spark.headless.keytab")
.set("spark.driver.extraLibraryPath",
"/usr/hdp/current/hadoop-client/lib/native:/usr/hdp/current/hadoop-client/lib/native/Linux-amd64-64")
.set("spark.yarn.queue", "default")
diff --git
a/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala
b/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala
index 86863f5..87a8f5d 100755
---
a/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala
+++
b/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala
@@ -16,10 +16,11 @@
*/
package org.apache.amaterasu.leader.mesos.schedulers
+import java.io.File
import java.util
import java.util.concurrent.locks.ReentrantLock
import java.util.concurrent.{ConcurrentHashMap, LinkedBlockingQueue}
-import java.util.{Collections, UUID}
+import java.util.{Collections, Properties, UUID}
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
@@ -50,6 +51,9 @@ import scala.collection.concurrent.TrieMap
*/
class JobScheduler extends AmaterasuScheduler {
+ /*private val props: Properties = new Properties(new File(""))
+ private val version = props.getProperty("version")
+ println(s"===> version $version")*/
LogManager.resetConfiguration()
private var jobManager: JobManager = _
private var client: CuratorFramework = _
@@ -166,15 +170,15 @@ class JobScheduler extends AmaterasuScheduler {
val command = CommandInfo
.newBuilder
.setValue(
- s"""$awsEnv env AMA_NODE=${sys.env("AMA_NODE")} env
MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so env
SPARK_EXECUTOR_URI=http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/dist/spark-${config.Webserver.sparkVersion}.tgz
java -cp
executor-0.2.0-incubating-all.jar:spark-${config.Webserver.sparkVersion}/jars/*
-Dscala.usejavacp=true -Djava.library.path=/usr/lib
org.apache.amaterasu.executor.mesos.executors.MesosActionsExecutor
${jobManager.jobId} ${config.master} ${ [...]
+ s"""$awsEnv env AMA_NODE=${sys.env("AMA_NODE")} env
MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so env
SPARK_EXECUTOR_URI=http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/dist/spark-${config.Webserver.sparkVersion}.tgz
java -cp
executor-${config.version}-all.jar:spark-${config.Webserver.sparkVersion}/jars/*
-Dscala.usejavacp=true -Djava.library.path=/usr/lib
org.apache.amaterasu.executor.mesos.executors.MesosActionsExecutor
${jobManager.jobId} ${config.master} $ [...]
)
-// HttpServer.getFilesInDirectory(sys.env("AMA_NODE"),
config.Webserver.Port).foreach(f=>
-// )
+ //
HttpServer.getFilesInDirectory(sys.env("AMA_NODE"),
config.Webserver.Port).foreach(f=>
+ // )
.addUris(URI.newBuilder
-
.setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/executor-0.2.0-incubating-all.jar")
- .setExecutable(false)
- .setExtract(false)
- .build())
+
.setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/executor-${config.version}-all.jar")
+ .setExecutable(false)
+ .setExtract(false)
+ .build())
.addUris(URI.newBuilder()
.setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/spark-2.2.1-bin-hadoop2.7.tgz")
.setExecutable(false)
@@ -201,10 +205,10 @@ class JobScheduler extends AmaterasuScheduler {
.setExtract(false)
.build())
.addUris(URI.newBuilder()
-
.setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/amaterasu.properties")
- .setExecutable(false)
- .setExtract(false)
- .build())
+
.setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/amaterasu.properties")
+ .setExecutable(false)
+ .setExtract(false)
+ .build())
executor = ExecutorInfo
.newBuilder
.setData(ByteString.copyFrom(execData))
diff --git
a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala
b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala
index 3fed076..8af5892 100644
---
a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala
+++
b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala
@@ -256,7 +256,7 @@ class ApplicationMaster extends
AMRMClientAsync.CallbackHandler with Logging {
val commands: List[String] = List(
"/bin/bash ./miniconda.sh -b -p $PWD/miniconda && ",
s"/bin/bash ${config.spark.home}/bin/load-spark-env.sh && ",
- s"java -cp
${config.spark.home}/jars/*:executor.jar:${config.spark.home}/conf/:${config.YARN.hadoopHomeDir}/conf/
" +
+ s"java -cp
executor.jar:${config.spark.home}/jars/*:${config.spark.home}/conf/:${config.YARN.hadoopHomeDir}/conf/
" +
"-Xmx1G " +
"-Dscala.usejavacp=true " +
"-Dhdp.version=2.6.1.0-129 " +
diff --git
a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala
b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala
index 0c1a8f0..9f4c3f9 100644
---
a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala
+++
b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala
@@ -106,7 +106,7 @@ class YarnRMCallbackHandler(nmClient: NMClientAsync,
val ctx = Records.newRecord(classOf[ContainerLaunchContext])
val command = s"""$awsEnv env AMA_NODE=${sys.env("AMA_NODE")}
| env
SPARK_EXECUTOR_URI=http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/dist/spark-${config.Webserver.sparkVersion}.tgz
- | java -cp
executor-0.2.0-all.jar:spark-${config.Webserver.sparkVersion}/lib/*
+ | java -cp
executor-*-all.jar:spark-${config.Webserver.sparkVersion}/lib/*
| -Dscala.usejavacp=true
| -Djava.library.path=/usr/lib
org.apache.amaterasu.executor.yarn.executors.ActionsExecutorLauncher
| ${jobManager.jobId} ${config.master}
${actionData.name} ${gson.toJson(taskData)}
${gson.toJson(execData)}""".stripMargin
diff --git a/leader/src/main/scripts/ama-start-yarn.sh
b/leader/src/main/scripts/ama-start-yarn.sh
index 6fd194a..0573341 100755
--- a/leader/src/main/scripts/ama-start-yarn.sh
+++ b/leader/src/main/scripts/ama-start-yarn.sh
@@ -86,7 +86,7 @@ done
echo "repo: ${REPO} "
echo "force-bin: ${FORCE_BIN}"
export HADOOP_USER_CLASSPATH_FIRST=true
-CMD="yarn jar ${BASEDIR}/bin/leader-0.2.0-incubating-all.jar
org.apache.amaterasu.leader.yarn.Client --home ${BASEDIR}"
+CMD="yarn jar ${BASEDIR}/bin/leader-0.2.0-incubating-rc2-all.jar
org.apache.amaterasu.leader.yarn.Client --home ${BASEDIR}"
if [ -n "$REPO" ]; then
echo "repo is ${REPO}"
diff --git a/leader/src/main/scripts/amaterasu.properties
b/leader/src/main/scripts/amaterasu.properties
index 7961db9..202e350 100755
--- a/leader/src/main/scripts/amaterasu.properties
+++ b/leader/src/main/scripts/amaterasu.properties
@@ -1,5 +1,5 @@
zk=127.0.0.1
-version=0.2.0-incubating
+version=0.2.0-incubating-rc2
master=192.168.33.11
user=root
mode=yarn
--
To stop receiving notification emails like this one, please contact
[email protected].
