Repository: spark Updated Branches: refs/heads/master 7f38b9d5f -> b4fbe140b
[SPARK-16349][SQL] Fall back to isolated class loader when classes not found. Some Hadoop classes needed by the Hive metastore client jars are not present in Spark's packaging (for example, "org/apache/hadoop/mapred/MRVersion"). So if the parent class loader fails to find a class, try to load it from the isolated class loader, in case it's available there. Tested by setting spark.sql.hive.metastore.jars to local paths with Hive/Hadoop libraries and verifying that Spark can talk to the metastore. Author: Marcelo Vanzin <van...@cloudera.com> Closes #14020 from vanzin/SPARK-16349. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b4fbe140 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b4fbe140 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b4fbe140 Branch: refs/heads/master Commit: b4fbe140be158f576706f21fa69f40d6e14e4a43 Parents: 7f38b9d Author: Marcelo Vanzin <van...@cloudera.com> Authored: Mon Jul 11 15:20:48 2016 -0700 Committer: Yin Huai <yh...@databricks.com> Committed: Mon Jul 11 15:20:48 2016 -0700 ---------------------------------------------------------------------- .../spark/sql/hive/client/IsolatedClientLoader.scala | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/b4fbe140/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala index e1950d1..a72a13b 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala @@ -220,9 +220,15 @@ private[hive] class IsolatedClientLoader( logDebug(s"hive class: $name - ${getResource(classToPath(name))}") super.loadClass(name, resolve) } else { - // For shared classes, we delegate to baseClassLoader. + // For shared classes, we delegate to baseClassLoader, but fall back in case the + // class is not found. logDebug(s"shared class: $name") - baseClassLoader.loadClass(name) + try { + baseClassLoader.loadClass(name) + } catch { + case _: ClassNotFoundException => + super.loadClass(name, resolve) + } } } } @@ -264,7 +270,7 @@ private[hive] class IsolatedClientLoader( throw new ClassNotFoundException( s"$cnf when creating Hive client using classpath: ${execJars.mkString(", ")}\n" + "Please make sure that jars for your version of hive and hadoop are included in the " + - s"paths passed to ${HiveUtils.HIVE_METASTORE_JARS}.", e) + s"paths passed to ${HiveUtils.HIVE_METASTORE_JARS.key}.", e) } else { throw e } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
