Repository: spark Updated Branches: refs/heads/master 76540b6df -> db5165246
[SPARK-11832][CORE] Process arguments in spark-shell for Scala 2.11 Process arguments passed to the spark-shell. Fixes running the spark-shell from within a build environment. Author: Jakob Odersky <[email protected]> Closes #9824 from jodersky/shell-2.11. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/db516524 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/db516524 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/db516524 Branch: refs/heads/master Commit: db5165246f2888537dd0f3d4c5a515875c7358ed Parents: 76540b6 Author: Jakob Odersky <[email protected]> Authored: Thu Dec 10 08:35:52 2015 -0800 Committer: Marcelo Vanzin <[email protected]> Committed: Thu Dec 10 08:35:52 2015 -0800 ---------------------------------------------------------------------- .../main/scala/org/apache/spark/repl/Main.scala | 37 ++++++++++++++------ .../scala/org/apache/spark/repl/ReplSuite.scala | 3 +- 2 files changed, 27 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/db516524/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala ---------------------------------------------------------------------- diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala index 627148d..455a6b9 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala @@ -31,24 +31,39 @@ object Main extends Logging { val tmp = System.getProperty("java.io.tmpdir") val rootDir = conf.get("spark.repl.classdir", tmp) val outputDir = Utils.createTempDir(rootDir) - val s = new Settings() - s.processArguments(List("-Yrepl-class-based", - "-Yrepl-outdir", s"${outputDir.getAbsolutePath}", - "-classpath", getAddedJars.mkString(File.pathSeparator)), true) // the creation of SecurityManager has to be lazy so SPARK_YARN_MODE is set if needed lazy val classServer = new HttpServer(conf, outputDir, new SecurityManager(conf)) var sparkContext: SparkContext = _ var sqlContext: SQLContext = _ var interp = new SparkILoop // this is a public var because tests reset it. + private var hasErrors = false + + private def scalaOptionError(msg: String): Unit = { + hasErrors = true + Console.err.println(msg) + } + def main(args: Array[String]) { - if (getMaster == "yarn-client") System.setProperty("SPARK_YARN_MODE", "true") - // Start the classServer and store its URI in a spark system property - // (which will be passed to executors so that they can connect to it) - classServer.start() - interp.process(s) // Repl starts and goes in loop of R.E.P.L - classServer.stop() - Option(sparkContext).map(_.stop) + + val interpArguments = List( + "-Yrepl-class-based", + "-Yrepl-outdir", s"${outputDir.getAbsolutePath}", + "-classpath", getAddedJars.mkString(File.pathSeparator) + ) ++ args.toList + + val settings = new Settings(scalaOptionError) + settings.processArguments(interpArguments, true) + + if (!hasErrors) { + if (getMaster == "yarn-client") System.setProperty("SPARK_YARN_MODE", "true") + // Start the classServer and store its URI in a spark system property + // (which will be passed to executors so that they can connect to it) + classServer.start() + interp.process(settings) // Repl starts and goes in loop of R.E.P.L + classServer.stop() + Option(sparkContext).map(_.stop) + } } def getAddedJars: Array[String] = { http://git-wip-us.apache.org/repos/asf/spark/blob/db516524/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala ---------------------------------------------------------------------- diff --git a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala index bf89979..63f3688 100644 --- a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala +++ b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala @@ -54,8 +54,7 @@ class ReplSuite extends SparkFunSuite { new SparkILoop(in, new PrintWriter(out)) } org.apache.spark.repl.Main.interp = interp - Main.s.processArguments(List("-classpath", classpath), true) - Main.main(Array()) // call main + Main.main(Array("-classpath", classpath)) // call main org.apache.spark.repl.Main.interp = null if (oldExecutorClasspath != null) { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
