Github user viirya commented on a diff in the pull request:
https://github.com/apache/spark/pull/8349#discussion_r38845757
--- Diff:
core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
---
@@ -398,30 +396,50 @@ private[spark] class MesosClusterScheduler(
throw new SparkException("Executor Spark home
`spark.mesos.executor.home` is not set!")
}
val cmdExecutable = new File(executorSparkHome,
"./bin/spark-submit").getCanonicalPath
- val cmdJar = desc.jarUrl.split("/").last
- (cmdExecutable, cmdJar)
+ // Sandbox points to the current directory by default with Mesos.
+ (cmdExecutable, ".")
}
- builder.setValue(s"$executable $cmdOptions $jar $appArguments")
+ val primaryResource = new File(sandboxPath,
desc.jarUrl.split("/").last).toString()
+ val cmdOptions = generateCmdOption(desc, sandboxPath).mkString(" ")
+ val appArguments = desc.command.arguments.mkString(" ")
+ builder.setValue(s"$executable $cmdOptions $primaryResource
$appArguments")
builder.setEnvironment(envBuilder.build())
conf.getOption("spark.mesos.uris").map { uris =>
setupUris(uris, builder)
}
+ desc.schedulerProperties.get("spark.mesos.uris").map { uris =>
+ setupUris(uris, builder)
+ }
+ desc.schedulerProperties.get("spark.submit.pyFiles").map { pyFiles =>
+ setupUris(pyFiles, builder)
+ }
builder.build()
}
- private def generateCmdOption(desc: MesosDriverDescription): Seq[String]
= {
+ private def generateCmdOption(desc: MesosDriverDescription, sandboxPath:
String): Seq[String] = {
var options = Seq(
"--name", desc.schedulerProperties("spark.app.name"),
- "--class", desc.command.mainClass,
"--master", s"mesos://${conf.get("spark.master")}",
"--driver-cores", desc.cores.toString,
"--driver-memory", s"${desc.mem}M")
+
+ if (!desc.command.mainClass.equals("")) {
+ // Assume empty main class means we're running python
--- End diff --
I don't use Python on Mesos, so if it is incorrect please correct me. When
we run python, don't we need to use `PythonRunner` as mainClass?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
