Github user marmbrus commented on a diff in the pull request:
https://github.com/apache/spark/pull/13201#discussion_r63973282
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
---
@@ -93,26 +101,45 @@ case class DataSource(
serviceLoader.asScala.filter(_.shortName().equalsIgnoreCase(provider)).toList
match {
// the provider format did not match any given registered aliases
case Nil =>
-
Try(loader.loadClass(provider)).orElse(Try(loader.loadClass(provider2))) match {
- case Success(dataSource) =>
- // Found the data source using fully qualified path
- dataSource
- case Failure(error) =>
- if (provider.startsWith("org.apache.spark.sql.hive.orc")) {
- throw new ClassNotFoundException(
- "The ORC data source must be used with Hive support
enabled.", error)
- } else {
- if (provider == "avro" || provider ==
"com.databricks.spark.avro") {
+ try {
+
Try(loader.loadClass(provider)).orElse(Try(loader.loadClass(provider2))) match {
+ case Success(dataSource) =>
+ // Found the data source using fully qualified path
+ dataSource
+ case Failure(error) =>
+ if (error.isInstanceOf[ClassNotFoundException]) {
+ val className = error.getMessage
+ if (spark2RemovedClasses.contains(className)) {
+ throw new ClassNotFoundException(s"$className is removed
in Spark 2.0. " +
+ "Please check if your library is compatible with Spark
2.0")
+ }
+ }
+ if (provider.startsWith("org.apache.spark.sql.hive.orc")) {
throw new ClassNotFoundException(
- s"Failed to find data source: $provider. Please use
Spark package " +
-
"http://spark-packages.org/package/databricks/spark-avro";,
- error)
+ "The ORC data source must be used with Hive support
enabled.", error)
} else {
- throw new ClassNotFoundException(
- s"Failed to find data source: $provider. Please find
packages at " +
- "http://spark-packages.org";,
- error)
+ if (provider == "avro" || provider ==
"com.databricks.spark.avro") {
+ throw new ClassNotFoundException(
+ s"Failed to find data source: $provider. Please use
Spark package " +
+
"http://spark-packages.org/package/databricks/spark-avro";,
+ error)
+ } else {
+ throw new ClassNotFoundException(
+ s"Failed to find data source: $provider. Please find
packages at " +
+ "http://spark-packages.org";,
+ error)
+ }
}
+ }
+ } catch {
+ case e: NoClassDefFoundError => // This one won't be caught by
Scala NonFatal
+ // NoClassDefFoundError's class name uses "/" rather than "."
for packages
+ val className = e.getMessage.replaceAll("/", ".")
+ if (spark2RemovedClasses.contains(className)) {
+ throw new ClassNotFoundException(s"$className is removed in
Spark 2.0. " +
+ "Please check if your library is compatible with Spark
2.0")
--- End diff --
Include `e` as the cause when throwing.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
