Repository: spark Updated Branches: refs/heads/branch-1.0 5c43758fb -> 6e0b7e530
[SPARK-1199][REPL] Remove VALId and use the original import style for defined classes. This is an alternate solution to #1176. Author: Prashant Sharma <[email protected]> Closes #1179 from ScrapCodes/SPARK-1199/repl-fix-second-approach and squashes the following commits: 820b34b [Prashant Sharma] Here we generate two kinds of import wrappers based on whether it is a class or not. (cherry picked from commit d43415075b3468fe8aa56de5d2907d409bb96347) Signed-off-by: Patrick Wendell <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6e0b7e53 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6e0b7e53 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6e0b7e53 Branch: refs/heads/branch-1.0 Commit: 6e0b7e5308263bef60120debe05577868ebaeea9 Parents: 5c43758 Author: Prashant Sharma <[email protected]> Authored: Fri Jul 4 00:05:27 2014 -0700 Committer: Patrick Wendell <[email protected]> Committed: Fri Jul 4 00:05:41 2014 -0700 ---------------------------------------------------------------------- .../org/apache/spark/repl/SparkIMain.scala | 7 ++++-- .../org/apache/spark/repl/SparkImports.scala | 23 ++++++++++++-------- .../scala/org/apache/spark/repl/ReplSuite.scala | 12 ++++++++++ 3 files changed, 31 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/6e0b7e53/repl/src/main/scala/org/apache/spark/repl/SparkIMain.scala ---------------------------------------------------------------------- diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkIMain.scala b/repl/src/main/scala/org/apache/spark/repl/SparkIMain.scala index 7c83fa9..3842c29 100644 --- a/repl/src/main/scala/org/apache/spark/repl/SparkIMain.scala +++ b/repl/src/main/scala/org/apache/spark/repl/SparkIMain.scala @@ -744,7 +744,7 @@ import org.apache.spark.util.Utils * * Read! Eval! Print! Some of that not yet centralized here. */ - class ReadEvalPrint(lineId: Int) { + class ReadEvalPrint(val lineId: Int) { def this() = this(freshLineId()) private var lastRun: Run = _ @@ -1241,7 +1241,10 @@ import org.apache.spark.util.Utils // old style beSilentDuring(parse(code)) foreach { ts => ts foreach { t => - withoutUnwrapping(logDebug(asCompactString(t))) + if (isShow || isShowRaw) + withoutUnwrapping(echo(asCompactString(t))) + else + withoutUnwrapping(logDebug(asCompactString(t))) } } } http://git-wip-us.apache.org/repos/asf/spark/blob/6e0b7e53/repl/src/main/scala/org/apache/spark/repl/SparkImports.scala ---------------------------------------------------------------------- diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkImports.scala b/repl/src/main/scala/org/apache/spark/repl/SparkImports.scala index 419796b..bce5c74 100644 --- a/repl/src/main/scala/org/apache/spark/repl/SparkImports.scala +++ b/repl/src/main/scala/org/apache/spark/repl/SparkImports.scala @@ -182,15 +182,26 @@ trait SparkImports { // ambiguity errors will not be generated. Also, quote // the name of the variable, so that we don't need to // handle quoting keywords separately. + case x: ClassHandler => + // I am trying to guess if the import is a defined class + // This is an ugly hack, I am not 100% sure of the consequences. + // Here we, let everything but "defined classes" use the import with val. + // The reason for this is, otherwise the remote executor tries to pull the + // classes involved and may fail. + for (imv <- x.definedNames) { + val objName = req.lineRep.readPath + code.append("import " + objName + ".INSTANCE" + req.accessPath + ".`" + imv + "`\n") + } + case x => for (imv <- x.definedNames) { if (currentImps contains imv) addWrapper() val objName = req.lineRep.readPath - val valName = "$VAL" + newValId(); + val valName = "$VAL" + req.lineRep.lineId if(!code.toString.endsWith(".`" + imv + "`;\n")) { // Which means already imported - code.append("val " + valName + " = " + objName + ".INSTANCE;\n") - code.append("import " + valName + req.accessPath + ".`" + imv + "`;\n") + code.append("val " + valName + " = " + objName + ".INSTANCE;\n") + code.append("import " + valName + req.accessPath + ".`" + imv + "`;\n") } // code.append("val " + valName + " = " + objName + ".INSTANCE;\n") // code.append("import " + valName + req.accessPath + ".`" + imv + "`;\n") @@ -211,10 +222,4 @@ trait SparkImports { private def membersAtPickler(sym: Symbol): List[Symbol] = beforePickler(sym.info.nonPrivateMembers.toList) - private var curValId = 0 - - private def newValId(): Int = { - curValId += 1 - curValId - } } http://git-wip-us.apache.org/repos/asf/spark/blob/6e0b7e53/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala ---------------------------------------------------------------------- diff --git a/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala index 95e1793..edd5243 100644 --- a/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala +++ b/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala @@ -225,6 +225,18 @@ class ReplSuite extends FunSuite { assertContains("res4: Array[Int] = Array(0, 0, 0, 0, 0)", output) } + test("SPARK-1199-simple-reproduce") { + val output = runInterpreter("local-cluster[1,1,512]", + """ + |case class Sum(exp: String, exp2: String) + |val a = Sum("A", "B") + |def b(a: Sum): String = a match { case Sum(_, _) => "Found Sum" } + |b(a) + """.stripMargin) + assertDoesNotContain("error:", output) + assertDoesNotContain("Exception", output) + } + if (System.getenv("MESOS_NATIVE_LIBRARY") != null) { test("running on Mesos") { val output = runInterpreter("localquiet",
