Repository: zeppelin Updated Branches: refs/heads/master 578fdf3e0 -> 3f5a500cc
[ZEPPELIN-1466] Make %dep work for spark 2.0 when SPARK_HOME is not defined ### What is this PR for? %dep does not work for spark 2.0 when SPARK_HOME is not defined. Problem described in the discussion https://issues.apache.org/jira/browse/ZEPPELIN-1466 ### What type of PR is it? Bug Fix ### Todos * [x] - Construct classpath arg correctly ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-1466 ### How should this be tested? make sure SPARK_HOME is NOT defined. run ``` %dep z.load("org.apache.commons:commons-csv:1.1") ``` and ``` %spark import org.apache.commons.csv.CSVParser ``` sequentially and see import statement success ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Lee moon soo <[email protected]> Closes #1481 from Leemoonsoo/ZEPPELIN-1466 and squashes the following commits: 5cdf99f [Lee moon soo] pass depInterpreter loaded jar in scala compiler's classpath arg Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/3f5a500c Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/3f5a500c Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/3f5a500c Branch: refs/heads/master Commit: 3f5a500ccf0f540fe2d9d488193a94f1eb737a3f Parents: 578fdf3 Author: Lee moon soo <[email protected]> Authored: Tue Oct 4 18:28:30 2016 +0900 Committer: Mina Lee <[email protected]> Committed: Thu Oct 6 10:47:25 2016 +0900 ---------------------------------------------------------------------- .../apache/zeppelin/spark/SparkInterpreter.java | 52 ++++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3f5a500c/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 44c2a74..dccc12e 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -584,6 +584,24 @@ public class SparkInterpreter extends Interpreter { argList.add(arg); } + DepInterpreter depInterpreter = getDepInterpreter(); + String depInterpreterClasspath = ""; + if (depInterpreter != null) { + SparkDependencyContext depc = depInterpreter.getDependencyContext(); + if (depc != null) { + List<File> files = depc.getFiles(); + if (files != null) { + for (File f : files) { + if (depInterpreterClasspath.length() > 0) { + depInterpreterClasspath += File.pathSeparator; + } + depInterpreterClasspath += f.getAbsolutePath(); + } + } + } + } + + if (Utils.isScala2_10()) { scala.collection.immutable.List<String> list = JavaConversions.asScalaBuffer(argList).toList(); @@ -611,10 +629,22 @@ public class SparkInterpreter extends Interpreter { argList.add("-Yrepl-class-based"); argList.add("-Yrepl-outdir"); argList.add(outputDir.getAbsolutePath()); + + String classpath = ""; if (conf.contains("spark.jars")) { - String jars = StringUtils.join(conf.get("spark.jars").split(","), File.separator); + classpath = StringUtils.join(conf.get("spark.jars").split(","), File.separator); + } + + if (!depInterpreterClasspath.isEmpty()) { + if (!classpath.isEmpty()) { + classpath += File.separator; + } + classpath += depInterpreterClasspath; + } + + if (!classpath.isEmpty()) { argList.add("-classpath"); - argList.add(jars); + argList.add(classpath); } scala.collection.immutable.List<String> list = @@ -626,6 +656,7 @@ public class SparkInterpreter extends Interpreter { // set classpath for scala compiler PathSetting pathSettings = settings.classpath(); String classpath = ""; + List<File> paths = currentClassPath(); for (File f : paths) { if (classpath.length() > 0) { @@ -644,21 +675,10 @@ public class SparkInterpreter extends Interpreter { } // add dependency from DepInterpreter - DepInterpreter depInterpreter = getDepInterpreter(); - if (depInterpreter != null) { - SparkDependencyContext depc = depInterpreter.getDependencyContext(); - if (depc != null) { - List<File> files = depc.getFiles(); - if (files != null) { - for (File f : files) { - if (classpath.length() > 0) { - classpath += File.pathSeparator; - } - classpath += f.getAbsolutePath(); - } - } - } + if (classpath.length() > 0) { + classpath += File.pathSeparator; } + classpath += depInterpreterClasspath; // add dependency from local repo String localRepo = getProperty("zeppelin.interpreter.localRepo");
