Repository: spark Updated Branches: refs/heads/branch-1.0 ade47562b -> 00fac73f7
SPARK-1746: Support setting SPARK_JAVA_OPTS on executors for backwards compatibility Author: Patrick Wendell <[email protected]> Closes #676 from pwendell/worker-opts and squashes the following commits: 54456c4 [Patrick Wendell] SPARK-1746: Support setting SPARK_JAVA_OPTS on executors for backwards compatibility (cherry picked from commit 913a0a9c0a87e164723ebf9616b883b6329bac71) Signed-off-by: Patrick Wendell <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/00fac73f Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/00fac73f Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/00fac73f Branch: refs/heads/branch-1.0 Commit: 00fac73f7f4119eef6ca971da1e8c4f6b07749ad Parents: ade4756 Author: Patrick Wendell <[email protected]> Authored: Wed May 7 00:11:05 2014 -0700 Committer: Patrick Wendell <[email protected]> Committed: Wed May 7 00:11:16 2014 -0700 ---------------------------------------------------------------------- .../org/apache/spark/deploy/worker/CommandUtils.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/00fac73f/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala index 3e615e7..c7f0f24 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala @@ -47,9 +47,16 @@ object CommandUtils extends Logging { */ def buildJavaOpts(command: Command, memory: Int, sparkHome: String): Seq[String] = { val memoryOpts = Seq(s"-Xms${memory}M", s"-Xmx${memory}M") - // Note, this will coalesce multiple options into a single command component val extraOpts = command.extraJavaOptions.map(Utils.splitCommandString).getOrElse(Seq()) + // Exists for backwards compatibility with older Spark versions + val workerLocalOpts = Option(getenv("SPARK_JAVA_OPTS")).map(Utils.splitCommandString) + .getOrElse(Nil) + if (workerLocalOpts.length > 0) { + logWarning("SPARK_JAVA_OPTS was set on the worker. It is deprecated in Spark 1.0.") + logWarning("Set SPARK_LOCAL_DIRS for node-specific storage locations.") + } + val libraryOpts = if (command.libraryPathEntries.size > 0) { val joined = command.libraryPathEntries.mkString(File.pathSeparator) @@ -66,7 +73,7 @@ object CommandUtils extends Logging { val userClassPath = command.classPathEntries ++ Seq(classPath) Seq("-cp", userClassPath.filterNot(_.isEmpty).mkString(File.pathSeparator)) ++ - libraryOpts ++ extraOpts ++ memoryOpts + libraryOpts ++ extraOpts ++ workerLocalOpts ++ memoryOpts } /** Spawn a thread that will redirect a given stream to a file */
