(paimon) branch master updated: [spark] Expose external table type to Spark (#8306)

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 5d3752ce88 [spark] Expose external table type to Spark (#8306)
5d3752ce88 is described below

commit 5d3752ce88c5166f5ab4679d75f1c76eaa24af94
Author: huangxiaoping <1754789...@qq.com>
AuthorDate: Wed Jun 24 14:48:45 2026 +0800

    [spark] Expose external table type to Spark (#8306)
    
    Fix an inconsistency where Spark did not recognize Paimon external
    tables as external in Spark-facing metadata.
    Before this change, a Paimon table could already be stored as
    `table.type=EXTERNAL`, but Spark-side logic relies on the reserved
    property `TableCatalog.PROP_EXTERNAL` to classify a table as external.
    Because that property was not exposed from `PaimonSparkTableBase`, Spark
    metadata output could report the wrong table type, such as showing a
    Paimon external table as managed in `DESC FORMATTED.`
    This change fixes that by exposing `PROP_EXTERNAL` from
    `PaimonSparkTableBase` when the underlying Paimon table is external.
    That keeps Spark behavior consistent with the existing property-based
    contract already used by Spark-side commands like `SHOW TABLE EXTENDED.`
---
 .../apache/paimon/spark/PaimonSparkTableBase.scala |  9 +++++++-
 .../paimon/spark/SparkCatalogWithHiveTest.java     | 24 ++++++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git 
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala
 
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala
index bd69174be7..6be314cee8 100644
--- 
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala
+++ 
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala
@@ -20,13 +20,14 @@ package org.apache.paimon.spark
 
 import org.apache.paimon.CoreOptions
 import org.apache.paimon.CoreOptions.BucketFunctionType
+import org.apache.paimon.options.CatalogOptions.TABLE_TYPE
 import org.apache.paimon.options.Options
 import org.apache.paimon.spark.catalog.functions.BucketFunction
 import org.apache.paimon.spark.read.PaimonSplitScanBuilder
 import org.apache.paimon.spark.schema.PaimonMetadataColumn
 import org.apache.paimon.spark.util.OptionUtils
 import org.apache.paimon.spark.write.{PaimonV2WriteBuilder, PaimonWriteBuilder}
-import org.apache.paimon.table.{Table, _}
+import org.apache.paimon.table.{CatalogTableType, Table, _}
 import org.apache.paimon.table.BucketMode.{BUCKET_UNAWARE, HASH_FIXED, 
POSTPONE_MODE}
 
 import org.apache.spark.sql.connector.catalog._
@@ -84,6 +85,12 @@ abstract class PaimonSparkTableBase(val table: Table)
         if (properties.containsKey(CoreOptions.PATH.key())) {
           properties.put(TableCatalog.PROP_LOCATION, 
properties.get(CoreOptions.PATH.key()))
         }
+        if (
+          CatalogTableType.EXTERNAL.toString.equalsIgnoreCase(
+            dataTable.options().get(TABLE_TYPE.key()))
+        ) {
+          properties.put(TableCatalog.PROP_EXTERNAL, "true")
+        }
         properties
       case _ => Collections.emptyMap()
     }
diff --git 
a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java
 
b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java
index 583522822b..f71e5df3c0 100644
--- 
a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java
+++ 
b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java
@@ -156,6 +156,30 @@ public class SparkCatalogWithHiveTest {
         }
     }
 
+    @Test
+    public void testDescribeExternalAndManagedTableType() throws IOException {
+        try (SparkSession spark = createSessionBuilder().getOrCreate()) {
+            spark.sql("CREATE DATABASE IF NOT EXISTS type_test_db");
+            spark.sql("USE spark_catalog.type_test_db");
+
+            spark.sql("CREATE EXTERNAL TABLE external_type_table (a INT, bb 
INT, c STRING)");
+            assertThat(
+                            spark.sql("DESC FORMATTED external_type_table")
+                                    .filter("col_name = 'Type'")
+                                    .head()
+                                    .getString(1))
+                    .isEqualTo("EXTERNAL");
+
+            spark.sql("CREATE TABLE managed_type_table (a INT)");
+            assertThat(
+                            spark.sql("DESC FORMATTED managed_type_table")
+                                    .filter("col_name = 'Type'")
+                                    .head()
+                                    .getString(1))
+                    .isEqualTo("MANAGED");
+        }
+    }
+
     private SparkSession.Builder createSessionBuilder() {
         Path warehousePath = new Path("file:" + tempDir.toString());
         return SparkSession.builder()