Github user marmbrus commented on a diff in the pull request:
https://github.com/apache/spark/pull/13201#discussion_r63973212
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
---
@@ -93,26 +101,45 @@ case class DataSource(
serviceLoader.asScala.filter(_.shortName().equalsIgnoreCase(provider)).toList
match {
// the provider format did not match any given registered aliases
case Nil =>
-
Try(loader.loadClass(provider)).orElse(Try(loader.loadClass(provider2))) match {
- case Success(dataSource) =>
- // Found the data source using fully qualified path
- dataSource
- case Failure(error) =>
- if (provider.startsWith("org.apache.spark.sql.hive.orc")) {
- throw new ClassNotFoundException(
- "The ORC data source must be used with Hive support
enabled.", error)
- } else {
- if (provider == "avro" || provider ==
"com.databricks.spark.avro") {
+ try {
+
Try(loader.loadClass(provider)).orElse(Try(loader.loadClass(provider2))) match {
+ case Success(dataSource) =>
+ // Found the data source using fully qualified path
+ dataSource
+ case Failure(error) =>
+ if (error.isInstanceOf[ClassNotFoundException]) {
+ val className = error.getMessage
+ if (spark2RemovedClasses.contains(className)) {
+ throw new ClassNotFoundException(s"$className is removed
in Spark 2.0. " +
+ "Please check if your library is compatible with Spark
2.0")
+ }
+ }
+ if (provider.startsWith("org.apache.spark.sql.hive.orc")) {
throw new ClassNotFoundException(
- s"Failed to find data source: $provider. Please use
Spark package " +
-
"http://spark-packages.org/package/databricks/spark-avro";,
- error)
+ "The ORC data source must be used with Hive support
enabled.", error)
} else {
- throw new ClassNotFoundException(
- s"Failed to find data source: $provider. Please find
packages at " +
- "http://spark-packages.org";,
- error)
+ if (provider == "avro" || provider ==
"com.databricks.spark.avro") {
+ throw new ClassNotFoundException(
+ s"Failed to find data source: $provider. Please use
Spark package " +
+
"http://spark-packages.org/package/databricks/spark-avro";,
+ error)
+ } else {
+ throw new ClassNotFoundException(
+ s"Failed to find data source: $provider. Please find
packages at " +
+ "http://spark-packages.org";,
+ error)
+ }
}
+ }
+ } catch {
+ case e: NoClassDefFoundError => // This one won't be caught by
Scala NonFatal
+ // NoClassDefFoundError's class name uses "/" rather than "."
for packages
+ val className = e.getMessage.replaceAll("/", ".")
+ if (spark2RemovedClasses.contains(className)) {
+ throw new ClassNotFoundException(s"$className is removed in
Spark 2.0. " +
--- End diff --
nit `is -> was`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
