This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 481559eea6 [spark] Convert exceptions to NoSuchTable for file-format
namespace (#7818)
481559eea6 is described below
commit 481559eea6157a76b1168d28fa965b602c1c5657
Author: Zouxxyy <[email protected]>
AuthorDate: Thu May 14 00:20:41 2026 +0800
[spark] Convert exceptions to NoSuchTable for file-format namespace (#7818)
When using Paimon as the Spark catalog, `parquet.\`path\`` queries may
fail with non-existence
exceptions (e.g., permission denied). This change converts all
exceptions for file-format
namespaces to `NoSuchTableException`, allowing Spark's
`ResolveSQLOnFile` to handle them.
---
.../java/org/apache/paimon/spark/SparkCatalog.java | 26 ++++++++++++++++++++++
.../apache/paimon/spark/sql/PaimonQueryTest.scala | 10 +++++++++
2 files changed, 36 insertions(+)
diff --git
a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java
b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java
index c68e7768ab..159aa98e37 100644
---
a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java
+++
b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java
@@ -69,6 +69,9 @@ import
org.apache.spark.sql.connector.expressions.FieldReference;
import org.apache.spark.sql.connector.expressions.IdentityTransform;
import org.apache.spark.sql.connector.expressions.NamedReference;
import org.apache.spark.sql.connector.expressions.Transform;
+import org.apache.spark.sql.execution.datasources.DataSource;
+import org.apache.spark.sql.execution.datasources.FileFormat;
+import org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
@@ -677,6 +680,29 @@ public class SparkCatalog extends SparkBaseCatalog
}
} catch (Catalog.TableNotExistException e) {
throw new NoSuchTableException(ident);
+ } catch (Exception e) {
+ // For SQL-on-file queries (e.g. SELECT * FROM parquet.`path`),
+ // swallow the exception to let Spark's ResolveSQLOnFile handle it.
+ if (isFileFormatNamespace(ident)) {
+ throw new NoSuchTableException(ident);
+ }
+ throw e;
+ }
+ }
+
+ /** Check if the identifier's namespace refers to a Spark FileFormat data
source. */
+ private static boolean isFileFormatNamespace(Identifier ident) {
+ if (ident.namespace().length != 1) {
+ return false;
+ }
+ try {
+ SparkSession spark = SparkSession.active();
+ Class<?> cls =
+ DataSource.lookupDataSource(ident.namespace()[0],
spark.sessionState().conf());
+ return FileFormat.class.isAssignableFrom(cls)
+ || cls.newInstance() instanceof FileDataSourceV2;
+ } catch (Exception e) {
+ return false;
}
}
diff --git
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonQueryTest.scala
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonQueryTest.scala
index d381af6ded..f94313f5e7 100644
---
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonQueryTest.scala
+++
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonQueryTest.scala
@@ -429,6 +429,16 @@ class PaimonQueryTest extends PaimonSparkTestBase {
}
}
+ test("Paimon Query: select from parquet datasource path under paimon
catalog") {
+ withTable("T") {
+ spark.sql("CREATE TABLE T (id INT, name STRING) USING paimon")
+ spark.sql("INSERT INTO T VALUES (1, 'a')")
+
+ val bucketDir = loadTable("T").location() + "/bucket-0"
+ checkAnswer(spark.sql(s"SELECT * FROM parquet.`$bucketDir`"),
spark.sql("SELECT * FROM T"))
+ }
+ }
+
fileFormats.foreach {
fileFormat =>
test(s"Query ignore-corrupt-files: file.format=$fileFormat") {
