Repository: zeppelin Updated Branches: refs/heads/branch-0.6 45dc8d23b -> 4b8b36fdf
[ZEPPELIN-1518] Fix for: Lambda expressions are not working on CDH 5.7.x Spark ### What is this PR for? Lambda expressions are not working on CDH 5.7.x Spark because of the backported RpcEnv and elimination of class server. ### What type of PR is it? Bug Fix ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-1518 ### How should this be tested? The following paragraph should work with SPARK_HOME pointing to CDH 5.7x spark: `val rdd = sc.parallelize(Seq(1,2,3,4,5)) rdd.filter(_ > 3).count()` ### Questions: * Does the licenses files need update? n * Is there breaking changes for older versions? n * Does this needs documentation? n Author: Vitaly Polonetsky <[email protected]> Closes #1486 from mvitaly/branch-0.6 and squashes the following commits: 4c97181 [Vitaly Polonetsky] [ZEPPELIN-1518] Fix for: Lambda expressions are not working on CDH 5.7.x Spark Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/4b8b36fd Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/4b8b36fd Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/4b8b36fd Branch: refs/heads/branch-0.6 Commit: 4b8b36fdf4bd11743c2f2b09dedc8d075ddb91d6 Parents: 45dc8d2 Author: Vitaly Polonetsky <[email protected]> Authored: Tue Oct 4 21:29:25 2016 -0700 Committer: Mina Lee <[email protected]> Committed: Fri Oct 7 14:40:56 2016 +0900 ---------------------------------------------------------------------- .../org/apache/zeppelin/spark/SparkInterpreter.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4b8b36fd/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index ed6efe4..42c30b9 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -381,6 +381,7 @@ public class SparkInterpreter extends Interpreter { } String classServerUri = null; + String replClassOutputDirectory = null; try { // in case of spark 1.1x, spark 1.2x Method classServer = intp.getClass().getMethod("classServer"); @@ -404,6 +405,16 @@ public class SparkInterpreter extends Interpreter { } } + if (classServerUri == null) { + try { // for RcpEnv + Method getClassOutputDirectory = intp.getClass().getMethod("getClassOutputDirectory"); + File classOutputDirectory = (File) getClassOutputDirectory.invoke(intp); + replClassOutputDirectory = classOutputDirectory.getAbsolutePath(); + } catch (NoSuchMethodException | SecurityException | IllegalAccessException + | IllegalArgumentException | InvocationTargetException e) { + // continue + } + } if (Utils.isScala2_11()) { classServer = createHttpServer(outputDir); @@ -418,6 +429,10 @@ public class SparkInterpreter extends Interpreter { conf.set("spark.repl.class.uri", classServerUri); } + if (replClassOutputDirectory != null) { + conf.set("spark.repl.class.outputDir", replClassOutputDirectory); + } + if (jars.length > 0) { conf.setJars(jars); }
