This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 305cdc326f60 [SPARK-55957][SQL] Add 'DATA_SOURCE_NOT_FOUND' in
Catalog.ERROR_HANDLING_RULES
305cdc326f60 is described below
commit 305cdc326f602d1109dececd863258c6f4b22217
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Wed Mar 11 14:35:30 2026 +0900
[SPARK-55957][SQL] Add 'DATA_SOURCE_NOT_FOUND' in
Catalog.ERROR_HANDLING_RULES
### What changes were proposed in this pull request?
Add `"DATA_SOURCE_NOT_FOUND" -> ReturnPartialResults` to
`ListTable.ERROR_HANDLING_RULES` in the classic Catalog so that `listTables()`
returns partial results when some tables fail with DATA_SOURCE_NOT_FOUND
instead of failing the entire call.
### Why are the changes needed?
When a table in the session catalog has an invalid or missing provider
(e.g. class not on classpath), resolving it throws `DATA_SOURCE_NOT_FOUND`.
Without this rule, `listTables()` would fail entirely; with it, other tables
are still returned and the bad one is included as a placeholder (partial
result).
### Does this PR introduce _any_ user-facing change?
Yes. `spark.catalog.listTables()` (and overloads) no longer throw when one
table has a missing data source; they return partial results.
### How was this patch tested?
New test in `CatalogSuite`: `listTables returns partial results when one
table has DATA_SOURCE_NOT_FOUND`.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #54750 from HyukjinKwon/SPARK-55957.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../scala/org/apache/spark/sql/classic/Catalog.scala | 3 ++-
.../org/apache/spark/sql/internal/CatalogSuite.scala | 20 ++++++++++++++++++++
2 files changed, 22 insertions(+), 1 deletion(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala
b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala
index 0b41c47cbe31..251877842bce 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala
@@ -979,7 +979,8 @@ private[sql] object Catalog {
val ERROR_HANDLING_RULES: Map[String, ErrorHandlingAction] = Map(
"UNSUPPORTED_FEATURE.HIVE_TABLE_TYPE" -> ReturnPartialResults,
- "TABLE_OR_VIEW_NOT_FOUND" -> Skip
+ "TABLE_OR_VIEW_NOT_FOUND" -> Skip,
+ "DATA_SOURCE_NOT_FOUND" -> ReturnPartialResults
)
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
index ebfffc14b014..7e064a7dab90 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
@@ -1136,6 +1136,26 @@ class CatalogSuite extends SharedSparkSession with
AnalysisTest with BeforeAndAf
}
}
+ test("SPARK-55957: listTables returns partial results when a table has
DATA_SOURCE_NOT_FOUND") {
+ // Create a normal table (resolvable)
+ createTable("good_table")
+ // Create a table with a non-existent provider so resolution throws
DATA_SOURCE_NOT_FOUND
+ val badTableMeta = utils.newTable("bad_table", None).copy(
+ provider = Some("non.existent.ProviderClass"))
+ sessionCatalog.createTable(badTableMeta, ignoreIfExists = false)
+ // Without ERROR_HANDLING_RULES for DATA_SOURCE_NOT_FOUND, listTables()
would throw.
+ // With the fix, we get partial results: good_table fully resolved,
bad_table as placeholder.
+ val tables = spark.catalog.listTables().collect()
+ assert(tables.length == 2, s"Expected 2 tables, got:
${tables.map(_.name).toList}")
+ val names = tables.map(_.name).toSet
+ assert(names == Set("good_table", "bad_table"),
+ s"Expected good_table and bad_table, got: $names")
+ val goodTable = tables.find(_.name == "good_table").get
+ val badTable = tables.find(_.name == "bad_table").get
+ assert(goodTable.tableType != null, "good_table should be fully resolved")
+ assert(badTable.tableType == null, "bad_table should be partial result
(tableType null)")
+ }
+
private def getConstructorParameterValues(obj: DefinedByConstructorParams):
Seq[AnyRef] = {
ScalaReflection.getConstructorParameterNames(obj.getClass).map { name =>
obj.getClass.getMethod(name).invoke(obj)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
