LIVY-323. Fixed livy-repl doesn't start with a lot of spark packages. (#312)
Some Spark packages are depending on scala-reflect 2.11.0 and it conflicts with Spark's scala version 2.11.8. This's not Livy's fault actually but doesn't hurt to make Livy more fault tolerant. Since the scala-reflect jar with the correct version must already be in CLASSPATH, fixed livy-repl to not load user supplied scala-reflect jars. Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/07f6072f Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/07f6072f Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/07f6072f Branch: refs/heads/master Commit: 07f6072f9d90ec970a1f9fbaccf0452f3a783538 Parents: 221aa9c Author: Alex Man <alex-the-...@users.noreply.github.com> Authored: Sat Apr 8 17:12:37 2017 -0700 Committer: GitHub <nore...@github.com> Committed: Sat Apr 8 17:12:37 2017 -0700 ---------------------------------------------------------------------- .../src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala | 4 ++++ .../src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala | 4 ++++ 2 files changed, 8 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/07f6072f/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala ---------------------------------------------------------------------- diff --git a/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala b/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala index ec12929..5ef5491 100644 --- a/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala +++ b/repl/scala-2.10/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala @@ -92,6 +92,10 @@ class SparkInterpreter(conf: SparkConf, .filter { u => u.getProtocol == "file" && new File(u.getPath).isFile } // Livy rsc and repl are also in the extra jars list. Filter them out. .filterNot { u => Paths.get(u.toURI).getFileName.toString.startsWith("livy-") } + // Some bad spark packages depend on the wrong version of scala-reflect. Blacklist it. + .filterNot { u => + Paths.get(u.toURI).getFileName.toString.contains("org.scala-lang_scala-reflect") + } extraJarPath.foreach { p => debug(s"Adding $p to Scala interpreter's class path...") } sparkIMain.addUrlsToClassPath(extraJarPath: _*) http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/07f6072f/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala ---------------------------------------------------------------------- diff --git a/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala b/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala index bf2f680..6735b3a 100644 --- a/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala +++ b/repl/scala-2.11/src/main/scala/com/cloudera/livy/repl/SparkInterpreter.scala @@ -78,6 +78,10 @@ class SparkInterpreter(conf: SparkConf, .filter { u => u.getProtocol == "file" && new File(u.getPath).isFile } // Livy rsc and repl are also in the extra jars list. Filter them out. .filterNot { u => Paths.get(u.toURI).getFileName.toString.startsWith("livy-") } + // Some bad spark packages depend on the wrong version of scala-reflect. Blacklist it. + .filterNot { u => + Paths.get(u.toURI).getFileName.toString.contains("org.scala-lang_scala-reflect") + } extraJarPath.foreach { p => debug(s"Adding $p to Scala interpreter's class path...") } sparkILoop.addUrlsToClassPath(extraJarPath: _*)
