Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/21378#discussion_r191070427
--- Diff:
resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
---
@@ -418,17 +417,34 @@ private[spark] class MesosClusterScheduler(
envBuilder.build()
}
+ private def isContainerLocalAppJar(desc: MesosDriverDescription):
Boolean = {
+ val isLocalJar = desc.jarUrl.startsWith("local://")
+ val isContainerLocal =
desc.conf.getOption("spark.mesos.appJar.local.resolution.mode").exists {
+ case "container" => true
+ case "host" => false
+ case other =>
+ logWarning(s"Unknown spark.mesos.appJar.local.resolution.mode
$other, using host.")
+ false
+ }
+ isLocalJar && isContainerLocal
+ }
+
private def getDriverUris(desc: MesosDriverDescription):
List[CommandInfo.URI] = {
val confUris = List(conf.getOption("spark.mesos.uris"),
desc.conf.getOption("spark.mesos.uris"),
desc.conf.getOption("spark.submit.pyFiles")).flatMap(
_.map(_.split(",").map(_.trim))
).flatten
- val jarUrl = desc.jarUrl.stripPrefix("file:").stripPrefix("local:")
-
- ((jarUrl :: confUris) ++ getDriverExecutorURI(desc).toList).map(uri =>
-
CommandInfo.URI.newBuilder().setValue(uri.trim()).setCache(useFetchCache).build())
+ if (isContainerLocalAppJar(desc)) {
+ (confUris ++ getDriverExecutorURI(desc).toList).map(uri =>
+
CommandInfo.URI.newBuilder().setValue(uri.trim()).setCache(useFetchCache).build())
+ }
+ else {
--- End diff --
nit: -> `} else {`
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
