(auron) branch master updated: [AURON #2427] Report fallback reason for unsupported Iceberg scan data types (#2428)

[slfan1989]

This is an automated email from the ASF dual-hosted git repository.

slfan1989 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git


The following commit(s) were added to refs/heads/master by this push:
     new 14a9c6d0 [AURON #2427] Report fallback reason for unsupported Iceberg 
scan data types (#2428)
14a9c6d0 is described below

commit 14a9c6d0f04298d436e7836e36efc0b13ee3ce75
Author: Ming Wei <weimingd...@gmail.com>
AuthorDate: Sun Jul 26 13:05:33 2026 +0800

    [AURON #2427] Report fallback reason for unsupported Iceberg scan data 
types (#2428)
    
    # Which issue does this PR close?
    
    Closes #2427
    
    # Rationale for this change
    Auron Iceberg native scan can fall back when the Iceberg scan schema
    contains data types that are not supported by native scan conversion.
    
    Before this change, unsupported metadata columns already had a clear
    fallback reason, but unsupported data columns did not. This made it
    harder to diagnose why an Iceberg scan stayed on Spark execution.
    
    
    # What changes are included in this PR?
    Adds an Iceberg fallback reason for unsupported scan schema data types.
    Reports unsupported field names and Spark SQL types in the never-convert
    reason.
    Extends the existing unsupported decimal Iceberg scan test to check the
    fallback reason.
    
    
    # Are there any user-facing changes?
    No API changes. Users may see a clearer fallback reason when an Iceberg
    native scan is skipped because of unsupported data types.
    
    
    
    # How was this patch tested?
    UT.
    
    # Was this patch authored or co-authored using generative AI tooling?
    - [ ] Yes
    - [x] No
    
    Co-authored-by: Shilun Fan <slfan1...@apache.org>
    Signed-off-by: weimingdiit <weimingd...@gmail.com>
---
 .../spark/sql/auron/iceberg/IcebergScanSupport.scala  | 19 ++++++++++++++++++-
 .../auron/iceberg/AuronIcebergIntegrationSuite.scala  |  6 ++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git 
a/thirdparty/auron-iceberg/src/main/scala/org/apache/spark/sql/auron/iceberg/IcebergScanSupport.scala
 
b/thirdparty/auron-iceberg/src/main/scala/org/apache/spark/sql/auron/iceberg/IcebergScanSupport.scala
index 31d87793..e280ab1e 100644
--- 
a/thirdparty/auron-iceberg/src/main/scala/org/apache/spark/sql/auron/iceberg/IcebergScanSupport.scala
+++ 
b/thirdparty/auron-iceberg/src/main/scala/org/apache/spark/sql/auron/iceberg/IcebergScanSupport.scala
@@ -83,7 +83,14 @@ object IcebergScanSupport extends Logging {
     if (collectUnsupportedMetadataColumns(scan.readSchema, 
isChangelogScan).nonEmpty) {
       Some("Has per-row materialization (for example _pos).")
     } else {
-      None
+      val unsupportedFields = 
collectUnsupportedDataTypeFields(scan.readSchema, isChangelogScan)
+      if (unsupportedFields.nonEmpty) {
+        Some(
+          s"Unsupported Iceberg scan schema. Unsupported fields/types: " +
+            s"${unsupportedFields.mkString(", ")}.")
+      } else {
+        None
+      }
     }
   }
 
@@ -377,6 +384,16 @@ object IcebergScanSupport extends Logging {
         field.name
     }
 
+  private def collectUnsupportedDataTypeFields(
+      schema: StructType,
+      isChangelogScan: Boolean): Seq[String] =
+    schema.fields
+      .filterNot(field =>
+        isIcebergMetadataColumn(field.name, isChangelogScan) &&
+          !isSupportedMetadataColumn(field, isChangelogScan))
+      .filterNot(field => NativeConverters.isTypeSupported(field.dataType))
+      .map(field => s"${field.name}: ${field.dataType.catalogString}")
+
   private def isIcebergMetadataColumn(name: String, isChangelogScan: Boolean): 
Boolean =
     MetadataColumns.isMetadataColumn(name) ||
       (isChangelogScan && ChangelogMetadataColumnNames.contains(name))
diff --git 
a/thirdparty/auron-iceberg/src/test/scala/org/apache/auron/iceberg/AuronIcebergIntegrationSuite.scala
 
b/thirdparty/auron-iceberg/src/test/scala/org/apache/auron/iceberg/AuronIcebergIntegrationSuite.scala
index 6d342bd4..1140b67b 100644
--- 
a/thirdparty/auron-iceberg/src/test/scala/org/apache/auron/iceberg/AuronIcebergIntegrationSuite.scala
+++ 
b/thirdparty/auron-iceberg/src/test/scala/org/apache/auron/iceberg/AuronIcebergIntegrationSuite.scala
@@ -733,6 +733,12 @@ class AuronIcebergIntegrationSuite
       checkAnswer(df, Seq(Row(1, new java.math.BigDecimal("123.4500000000"))))
       val plan = df.queryExecution.executedPlan.toString()
       assert(!plan.contains("NativeIcebergTableScan"))
+      val neverConvertReasonTag: TreeNodeTag[String] = 
TreeNodeTag("auron.never.convert.reason")
+      assert(
+        collectFirst(df.queryExecution.executedPlan) { case batchScanExec: 
BatchScanExec =>
+          batchScanExec.getTagValue(neverConvertReasonTag)
+        }.get.get.equals(
+          "Unsupported Iceberg scan schema. Unsupported fields/types: amount: 
decimal(38,10)."))
     }
   }