This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.1 by this push:
     new dff6db382863 [SPARK-60796][SQL] Fix 
`HiveTableScanExec.prunePartitions` to treat the default partition name as null
dff6db382863 is described below

commit dff6db382863f5ee29cbc28f428e9e42b2892093
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sun Nov 1 16:05:23 2026 -0800

    [SPARK-60796][SQL] Fix `HiveTableScanExec.prunePartitions` to treat the 
default partition name as null
    
    ### What changes were proposed in this pull request?
    
    This PR aims to fix `HiveTableScanExec.prunePartitions` to treat the 
default partition name, `__HIVE_DEFAULT_PARTITION__`, as `NULL`, like 
`CatalogTablePartition.toRow` and `HiveShim` already do.
    
    `prunePartitions` casts the raw value of every partition column, including 
the columns which the pruning predicates do not reference. It is used in the 
following cases.
    
    - The relation is already pruned by `PruneHiveTablePartitions` and the 
predicates have an exec subquery, e.g. a dynamic pruning filter reusing a 
broadcast, a runtime bloom filter, or a scalar subquery.
    - The relation is not pruned yet and 
`spark.sql.hive.metastorePartitionPruning` is `false`.
    
    ### Why are the changes needed?
    
    The default partition name stands for a `NULL` partition value, but 
`prunePartitions` casts it like a regular value. This causes two issues.
    
    1. A non-string partition column fails the query in ANSI mode, even when 
the predicates do not reference the column.
    2. A string partition column keeps the name as a non-null value. `IS NULL` 
does not match it, so the partition is silently dropped and the query returns a 
wrong result regardless of the ANSI mode.
    
    ```sql
    SET hive.exec.dynamic.partition.mode=nonstrict;
    CREATE TABLE dim (k INT, x INT) STORED AS TEXTFILE;
    INSERT INTO dim VALUES (1, 1);
    CREATE TABLE t1 (a INT) PARTITIONED BY (p1 INT, p2 INT) STORED AS TEXTFILE;
    INSERT INTO t1 PARTITION (p1 = 1, p2) VALUES (10, NULL), (11, 5);
    CREATE TABLE t2 (a INT) PARTITIONED BY (p1 INT, p2 STRING) STORED AS 
TEXTFILE;
    INSERT INTO t2 PARTITION (p1 = 1, p2) VALUES (10, NULL), (11, 'x');
    
    -- Issue 1: It should return 10 and 11, but it fails.
    SELECT /*+ BROADCAST(d) */ t1.a FROM t1 JOIN (SELECT k FROM dim WHERE x = 
1) d ON t1.p1 = d.k;
    -- [CAST_INVALID_INPUT] The value '__HIVE_DEFAULT_PARTITION__' of the type 
"STRING" cannot be cast to "INT" ...
    
    -- Issue 2: It should return 10, but it returns no row.
    -- It returns 10 if `spark.sql.optimizer.dynamicPartitionPruning.enabled` 
is `false`.
    SELECT /*+ BROADCAST(d) */ t2.a FROM t2 JOIN (SELECT k FROM dim WHERE x = 
1) d ON t2.p1 = d.k
    WHERE t2.p2 IS NULL;
    ```
    
    The hint is not required for Issue 2. The same happens with the default 
configurations when the dimension table is the broadcast side at planning time, 
e.g. after `ANALYZE TABLE` on a dimension table and a larger fact table.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. The above queries return the correct results after this PR. Both 
issues are reproduced with Apache Spark 4.0.4, 4.1.3, 4.2.0, and 4.3.0.
    
    This PR does not change how `HadoopTableReader.fillPartitionKeys` fills the 
partition columns of the scan output. For example, `SELECT p2 FROM t1` still 
fails in ANSI mode, and `SELECT p2 FROM t2` still returns the name instead of 
`NULL`.
    
    ### How was this patch tested?
    
    Pass the CIs with the newly added test case.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    Closes #60002 from dongjoon-hyun/SPARK-60796.
    
    Authored-by: Dongjoon Hyun <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
    (cherry picked from commit 71a1c3f09f1865472a11e22153376a0676db14e1)
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 .../sql/hive/execution/HiveTableScanExec.scala     |  5 +--
 .../sql/hive/execution/HiveTableScanSuite.scala    | 40 +++++++++++++++++++++-
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git 
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
 
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
index 5e4ef82d798b..921c4ad67618 100644
--- 
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
+++ 
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala
@@ -33,7 +33,7 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.CastSupport
-import org.apache.spark.sql.catalyst.catalog.HiveTableRelation
+import org.apache.spark.sql.catalyst.catalog.{ExternalCatalogUtils, 
HiveTableRelation}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.execution._
@@ -112,7 +112,8 @@ case class HiveTableScanExec(
     hadoopConf)
 
   private def castFromString(value: String, dataType: DataType) = {
-    cast(Literal(value), dataType).eval(null)
+    val partValue = if (value == ExternalCatalogUtils.DEFAULT_PARTITION_NAME) 
null else value
+    cast(Literal(partValue), dataType).eval(null)
   }
 
   private def addColumnMetadataToConf(hiveConf: Configuration): Unit = {
diff --git 
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
 
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
index 033d86194bda..8bebb2d42037 100644
--- 
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
+++ 
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.hive.execution
 
 import java.io.{File, IOException}
 
-import org.apache.spark.sql.Row
+import org.apache.spark.sql.{QueryTest, Row}
 import 
org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
 import org.apache.spark.sql.functions.col
 import org.apache.spark.sql.hive.test.{TestHive, TestHiveSingleton}
@@ -278,6 +278,44 @@ class HiveTableScanSuite extends HiveComparisonTest with 
SQLTestUtils with TestH
     }
   }
 
+  test("SPARK-60796: Treat the default partition name as null when pruning 
partitions") {
+    withTable("fact", "dim") {
+      sql("CREATE TABLE dim (k STRING, x INT) USING hive")
+      sql("INSERT INTO dim VALUES ('1', 1)")
+      sql("CREATE TABLE fact (a INT) USING hive PARTITIONED BY (p1 STRING, p2 
INT)")
+      withSQLConf("hive.exec.dynamic.partition.mode" -> "nonstrict") {
+        sql(
+          """
+            |INSERT INTO fact PARTITION (p1, p2)
+            |VALUES (10, '1', NULL), (11, '1', 5), (12, NULL, 5), (13, '2', 5)
+          """.stripMargin)
+      }
+
+      // The dynamic pruning filter references only `p1`, but all the 
partition values are cast to
+      // evaluate it. `p2` of the default partition fails the ANSI cast unless 
it is null.
+      withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+        QueryTest.checkAnswer(
+          sql(
+            """
+              |SELECT /*+ BROADCAST(d) */ f.a
+              |FROM fact f JOIN (SELECT k FROM dim WHERE x = 1) d ON f.p1 = d.k
+            """.stripMargin),
+          Seq(Row(10), Row(11)))
+      }
+
+      // `PruneHiveTablePartitions` skips a filter with a subquery, so the 
scan has to prune all the
+      // listed partitions. No cast fails without ANSI, but `p1` of the row `a 
= 12` must still be
+      // null rather than the name to match `IS NULL`.
+      withSQLConf(
+        SQLConf.ANSI_ENABLED.key -> "false",
+        SQLConf.HIVE_METASTORE_PARTITION_PRUNING.key -> "false") {
+        QueryTest.checkAnswer(
+          sql("SELECT a FROM fact WHERE p1 IS NULL OR p1 = (SELECT max(k) FROM 
dim)"),
+          Seq(Row(10), Row(11), Row(12)))
+      }
+    }
+  }
+
   private def getHiveTableScanExec(query: String): HiveTableScanExec = {
     sql(query).queryExecution.sparkPlan.collectFirst {
       case p: HiveTableScanExec => p


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to