Repository: zeppelin Updated Branches: refs/heads/master 1da262829 -> 544aa3b66
[ZEPPELIN-1518] Fix for: Lambda expressions are not working on CDH 5.7.x Spark ### What is this PR for? Lambda expressions are not working on CDH 5.7.x Spark because of the backported RpcEnv and elimination of class server. ### What type of PR is it? Bug Fix ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-1518 ### How should this be tested? The following paragraph should work with SPARK_HOME pointing to CDH 5.7x spark: `val rdd = sc.parallelize(Seq(1,2,3,4,5)) rdd.filter(_ > 3).count()` ### Questions: * Does the licenses files need update? n * Is there breaking changes for older versions? n * Does this needs documentation? n Author: Vitaly Polonetsky <[email protected]> Closes #1486 from mvitaly/branch-0.6 and squashes the following commits: 4c97181 [Vitaly Polonetsky] [ZEPPELIN-1518] Fix for: Lambda expressions are not working on CDH 5.7.x Spark (cherry picked from commit 4b8b36fdf4bd11743c2f2b09dedc8d075ddb91d6) Signed-off-by: Mina Lee <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/544aa3b6 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/544aa3b6 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/544aa3b6 Branch: refs/heads/master Commit: 544aa3b66751e84646e7293f9c0d23a3aa372ae1 Parents: 1da2628 Author: Vitaly Polonetsky <[email protected]> Authored: Tue Oct 4 21:29:25 2016 -0700 Committer: Mina Lee <[email protected]> Committed: Fri Oct 7 14:41:11 2016 +0900 ---------------------------------------------------------------------- .../org/apache/zeppelin/spark/SparkInterpreter.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/544aa3b6/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index dccc12e..878c010 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -384,6 +384,7 @@ public class SparkInterpreter extends Interpreter { } String classServerUri = null; + String replClassOutputDirectory = null; try { // in case of spark 1.1x, spark 1.2x Method classServer = intp.getClass().getMethod("classServer"); @@ -407,6 +408,16 @@ public class SparkInterpreter extends Interpreter { } } + if (classServerUri == null) { + try { // for RcpEnv + Method getClassOutputDirectory = intp.getClass().getMethod("getClassOutputDirectory"); + File classOutputDirectory = (File) getClassOutputDirectory.invoke(intp); + replClassOutputDirectory = classOutputDirectory.getAbsolutePath(); + } catch (NoSuchMethodException | SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + // continue + } + } if (Utils.isScala2_11()) { classServer = createHttpServer(outputDir); @@ -421,6 +432,10 @@ public class SparkInterpreter extends Interpreter { conf.set("spark.repl.class.uri", classServerUri); } + if (replClassOutputDirectory != null) { + conf.set("spark.repl.class.outputDir", replClassOutputDirectory); + } + if (jars.length > 0) { conf.setJars(jars); }
