zjffdu commented on a change in pull request #3308: [ZEPPELIN-3986]. Cannot
access any JAR in yarn cluster mode
URL: https://github.com/apache/zeppelin/pull/3308#discussion_r257488151
##########
File path:
spark/spark-scala-parent/src/main/scala/org/apache/zeppelin/spark/BaseSparkScalaInterpreter.scala
##########
@@ -370,20 +374,25 @@ abstract class BaseSparkScalaInterpreter(val conf:
SparkConf,
}
protected def getUserJars(): Seq[String] = {
- val sparkJars = conf.getOption("spark.jars").map(_.split(","))
- .map(_.filter(_.nonEmpty)).toSeq.flatten
- val depJars = depFiles.asScala.filter(_.endsWith(".jar"))
- // add zeppelin spark interpreter jar
- val zeppelinInterpreterJarURL =
getClass.getProtectionDomain.getCodeSource.getLocation
- // zeppelinInterpreterJarURL might be a folder when under unit testing
- val result = if (new File(zeppelinInterpreterJarURL.getFile).isDirectory) {
- sparkJars ++ depJars
- } else {
- sparkJars ++ depJars ++ Seq(zeppelinInterpreterJarURL.getFile)
+ var classLoader = Thread.currentThread().getContextClassLoader
+ var extraJars = Seq.empty[String]
+ while (classLoader != null) {
+ if (classLoader.getClass.getCanonicalName ==
+ "org.apache.spark.util.MutableURLClassLoader") {
+ extraJars = classLoader.asInstanceOf[URLClassLoader].getURLs()
+ // Check if the file exists.
+ .filter { u => u.getProtocol == "file" && new File(u.getPath).isFile
}
+ // Some bad spark packages depend on the wrong version of
scala-reflect. Blacklist it.
+ .filterNot { u =>
+
Paths.get(u.toURI).getFileName.toString.contains("org.scala-lang_scala-reflect")
Review comment:
User jar is configurable via `spark.jars` or `spark.jars.packages`. Here's
the internal mechnism at runtime of detecting what user jars user has been
specified
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
