Repository: spark Updated Branches: refs/heads/master 6b3c6e5dd -> 71f4d2612
Fix SPARK-1609: Executor fails to start when Command.extraJavaOptions contains multiple Java options Author: witgo <[email protected]> Closes #547 from witgo/SPARK-1609 and squashes the following commits: deb6a4c [witgo] review commit 91da0bb [witgo] Merge branch 'master' of https://github.com/apache/spark into SPARK-1609 0640852 [witgo] review commit 8f90b22 [witgo] Merge branch 'master' of https://github.com/apache/spark into SPARK-1609 bcf36cb [witgo] Merge branch 'master' of https://github.com/apache/spark into SPARK-1609 1185605 [witgo] fix extraJavaOptions split f7c0ab7 [witgo] bugfix 86fc4bb [witgo] bugfix 8a265b7 [witgo] Fix SPARK-1609: Executor fails to start when use spark-submit Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/71f4d261 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/71f4d261 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/71f4d261 Branch: refs/heads/master Commit: 71f4d2612a1be2904ed1536280680abc2dd212e7 Parents: 6b3c6e5 Author: witgo <[email protected]> Authored: Sun Apr 27 19:41:02 2014 -0700 Committer: Patrick Wendell <[email protected]> Committed: Sun Apr 27 19:41:02 2014 -0700 ---------------------------------------------------------------------- .../scala/org/apache/spark/deploy/worker/CommandUtils.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/71f4d261/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala index 9103c88..3e615e7 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala @@ -48,7 +48,8 @@ object CommandUtils extends Logging { def buildJavaOpts(command: Command, memory: Int, sparkHome: String): Seq[String] = { val memoryOpts = Seq(s"-Xms${memory}M", s"-Xmx${memory}M") // Note, this will coalesce multiple options into a single command component - val extraOpts = command.extraJavaOptions.toSeq + val extraOpts = command.extraJavaOptions.map(Utils.splitCommandString).getOrElse(Seq()) + val libraryOpts = if (command.libraryPathEntries.size > 0) { val joined = command.libraryPathEntries.mkString(File.pathSeparator) @@ -62,10 +63,10 @@ object CommandUtils extends Logging { val classPath = Utils.executeAndGetOutput( Seq(sparkHome + "/bin/compute-classpath" + ext), extraEnvironment=command.environment) - val userClassPath = command.classPathEntries.mkString(File.pathSeparator) - val classPathWithUser = classPath + File.pathSeparator + userClassPath + val userClassPath = command.classPathEntries ++ Seq(classPath) - Seq("-cp", classPathWithUser) ++ libraryOpts ++ extraOpts ++ memoryOpts + Seq("-cp", userClassPath.filterNot(_.isEmpty).mkString(File.pathSeparator)) ++ + libraryOpts ++ extraOpts ++ memoryOpts } /** Spawn a thread that will redirect a given stream to a file */
