This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 2db9d78d716 [SPARK-39174][SQL] Catalogs loading swallows missing
classname for ClassNotFoundException
2db9d78d716 is described below
commit 2db9d78d71662de276dd65e582674b6088eff119
Author: Kent Yao <[email protected]>
AuthorDate: Sat May 14 19:45:24 2022 +0900
[SPARK-39174][SQL] Catalogs loading swallows missing classname for
ClassNotFoundException
### What changes were proposed in this pull request?
this PR captures the actual missing classname when catalog loading meets
ClassNotFoundException
### Why are the changes needed?
ClassNotFoundException can occur when missing dependencies, we shall not
always report the catalog class is missing
### Does this PR introduce _any_ user-facing change?
yes, when loading catalogs and ClassNotFoundException occurs, it shows the
correct missing class.
### How was this patch tested?
new test added
Closes #36534 from yaooqinn/SPARK-39174.
Authored-by: Kent Yao <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
(cherry picked from commit 1b37f19876298e995596a30edc322c856ea1bbb4)
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../spark/sql/connector/catalog/Catalogs.scala | 5 ++--
.../spark/sql/errors/QueryExecutionErrors.scala | 5 ++--
.../sql/connector/catalog/CatalogLoadingSuite.java | 28 ++++++++++++++++++++++
3 files changed, 34 insertions(+), 4 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala
index 9949f45d483..71b1042ab30 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala
@@ -60,8 +60,9 @@ private[sql] object Catalogs {
plugin.initialize(name, catalogOptions(name, conf))
plugin
} catch {
- case _: ClassNotFoundException =>
- throw
QueryExecutionErrors.catalogPluginClassNotFoundForCatalogError(name,
pluginClassName)
+ case e: ClassNotFoundException =>
+ throw QueryExecutionErrors.catalogPluginClassNotFoundForCatalogError(
+ name, pluginClassName, e)
case e: NoSuchMethodException =>
throw
QueryExecutionErrors.catalogFailToFindPublicNoArgConstructorError(
name, pluginClassName, e)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
index 7825e9a94dc..df5959283eb 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
@@ -1586,8 +1586,9 @@ object QueryExecutionErrors extends QueryErrorsBase {
def catalogPluginClassNotFoundForCatalogError(
name: String,
- pluginClassName: String): Throwable = {
- new SparkException(s"Cannot find catalog plugin class for catalog '$name':
$pluginClassName")
+ pluginClassName: String,
+ e: Exception): Throwable = {
+ new SparkException(s"Cannot find catalog plugin class for catalog '$name':
$pluginClassName", e)
}
def catalogFailToFindPublicNoArgConstructorError(
diff --git
a/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java
b/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java
index 37f60511cd6..81870508b70 100644
---
a/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java
+++
b/sql/catalyst/src/test/java/org/apache/spark/sql/connector/catalog/CatalogLoadingSuite.java
@@ -20,6 +20,8 @@ package org.apache.spark.sql.connector.catalog;
import org.apache.spark.SparkException;
import org.apache.spark.sql.internal.SQLConf;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
+import org.apache.spark.util.Utils;
+
import org.junit.Assert;
import org.junit.Test;
@@ -91,6 +93,19 @@ public class CatalogLoadingSuite {
exc.getMessage().contains("com.example.NoSuchCatalogPlugin"));
}
+ @Test
+ public void testLoadMissingDependentClasses() {
+ SQLConf conf = new SQLConf();
+ String catalogClass = ClassFoundCatalogPlugin.class.getCanonicalName();
+ conf.setConfString("spark.sql.catalog.missing", catalogClass);
+
+ SparkException exc =
+ Assert.assertThrows(SparkException.class, () ->
Catalogs.load("missing", conf));
+
+ Assert.assertTrue(exc.getCause() instanceof ClassNotFoundException);
+ Assert.assertTrue(exc.getCause().getMessage().contains(catalogClass +
"Dep"));
+ }
+
@Test
public void testLoadNonCatalogPlugin() {
SQLConf conf = new SQLConf();
@@ -209,3 +224,16 @@ class InvalidCatalogPlugin { // doesn't implement
CatalogPlugin
public void initialize(CaseInsensitiveStringMap options) {
}
}
+
+class ClassFoundCatalogPlugin implements CatalogPlugin {
+
+ @Override
+ public void initialize(String name, CaseInsensitiveStringMap options) {
+ Utils.classForName(this.getClass().getCanonicalName() + "Dep", true, true);
+ }
+
+ @Override
+ public String name() {
+ return null;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
