This is an automated email from the ASF dual-hosted git repository.

szehon-ho pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.1 by this push:
     new 622757ac9b76 [SPARK-60486][4.1][SQL] Preserve Python UDF partition 
filters after V2 pushdown
622757ac9b76 is described below

commit 622757ac9b7620ed9374ead13423e313f6209b99
Author: 小哥不太逍遥 <[email protected]>
AuthorDate: Thu Oct 29 13:53:24 2026 -0700

    [SPARK-60486][4.1][SQL] Preserve Python UDF partition filters after V2 
pushdown
    
    ### What changes were proposed in this pull request?
    
    This backports the accepted [SPARK-60486 fix from 
#59732](https://github.com/apache/spark/pull/59732) to branch-4.1. Partition 
predicates that cannot be pushed into the scan are returned for post-scan 
evaluation, while this branch's existing data-filter translation loop is 
preserved. The regression test covers a Python UDF predicate on a partition 
column alone and in AND/OR combinations; the migration guide records the 
behavior change.
    
    ### Why are the changes needed?
    
    `FileScanBuilder` runs before `ExtractPythonUDFs`, so a Python UDF 
partition predicate cannot be pushed into the scan. Previously, the builder 
excluded it from pushdown without returning it as a residual filter, which 
could omit the predicate and return rows that do not satisfy the query. Keeping 
it as a post-scan filter preserves SQL filter semantics.
    
    ### Does this PR introduce any user-facing change?
    
    Yes. Data Source V2 file scans now evaluate deterministic Python UDF 
predicates over partition columns. Previously, such predicates could be dropped 
and return extra rows. The 4.1 migration guide notes that data derived from 
affected query results may need to be recomputed.
    
    ### How was this patch tested?
    
    - Fork CI run 
[40813160703](https://github.com/xgbtxy/spark/actions/runs/40813160703) passed 
Precompile Spark and PySpark SQL. Both 
`UDFTests.test_file_dsv2_with_udf_filter` and 
`ArrowPythonUDFTests.test_file_dsv2_with_udf_filter` passed with no skips; 
`PruneFileSourcePartitionsSuite` passed 9/9 with no skips.
    - The [full fork workflow passed on attempt 
2](https://github.com/xgbtxy/spark/actions/runs/40813160703/attempts/2) on 
unchanged commit `a2c77e5`, after rerunning the SQL-other job.
    - Black, flake8, and the branch-pinned single-file scalafmt check passed.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex (GPT-6)
    
    Closes #59904 from xgbtxy/backport/spark-60486-4.1.
    
    Authored-by: 小哥不太逍遥 <[email protected]>
    Signed-off-by: Szehon Ho <[email protected]>
---
 docs/sql-migration-guide.md                         |  4 ++++
 python/pyspark/sql/tests/test_udf.py                | 21 +++++++++++++++++++--
 .../execution/datasources/v2/FileScanBuilder.scala  |  7 ++++---
 3 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 0a2533d28f0b..e2665a0a89a0 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -22,6 +22,10 @@ license: |
 * Table of contents
 {:toc}
 
+## Upgrading from Spark SQL 4.1.4 to 4.1.5
+
+- Since Spark 4.1.5, Data Source V2 file scans correctly evaluate 
deterministic Python UDF filter predicates that reference only partition 
columns. Previously, these predicates could be dropped, returning rows that did 
not satisfy the filter (SPARK-60486). Data derived from affected query results 
may need to be recomputed.
+
 ## Upgrading from Spark SQL 4.0 to 4.1
 
 - Since Spark 4.1, the Parquet reader no longer assumes all struct values to 
be null, if all the requested fields are missing in the parquet file. The new 
default behavior is to read an additional struct field that is present in the 
file to determine nullness. To restore the previous behavior, set 
`spark.sql.legacy.parquet.returnNullStructIfAllFieldsMissing` to `true`.
diff --git a/python/pyspark/sql/tests/test_udf.py 
b/python/pyspark/sql/tests/test_udf.py
index 9c5fa2ad1bba..de4ce5c2761b 100644
--- a/python/pyspark/sql/tests/test_udf.py
+++ b/python/pyspark/sql/tests/test_udf.py
@@ -886,12 +886,29 @@ class BaseUDFTestsMixin(object):
 
         try:
             with self.sql_conf({"spark.sql.sources.useV1SourceList": ""}):
-                
self.spark.range(1).write.mode("overwrite").format("parquet").save(path)
-                df = self.spark.read.parquet(path).toDF("i")
+                (
+                    self.spark.range(6)
+                    .selectExpr("id AS i", "id % 3 AS p")
+                    .write.mode("overwrite")
+                    .partitionBy("p")
+                    .parquet(path)
+                )
+                df = self.spark.read.parquet(path)
                 f = udf(lambda x: False, "boolean")(lit(1))
                 result = df.filter(f)
                 self.assertEqual(0, result.count())
 
+                # SPARK-60486: partition predicates that cannot be pushed must 
still be evaluated.
+                matches_partition = udf(lambda x: x == 1, "boolean")(col("p"))
+                expected = [Row(i=1, p=1), Row(i=4, p=1)]
+                assertDataFrameEqual(df.filter(col("p") == 1), expected)
+                assertDataFrameEqual(df.filter(matches_partition), expected)
+                assertDataFrameEqual(df.filter(matches_partition & (col("p") > 
0)), expected)
+                assertDataFrameEqual(
+                    df.filter(matches_partition | (col("p") == 0)),
+                    [Row(i=0, p=0), Row(i=1, p=1), Row(i=3, p=0), Row(i=4, 
p=1)],
+                )
+
         finally:
             shutil.rmtree(path)
 
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileScanBuilder.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileScanBuilder.scala
index 7e0bc25a9a1e..14f6192431f0 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileScanBuilder.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileScanBuilder.scala
@@ -31,7 +31,7 @@ abstract class FileScanBuilder(
     sparkSession: SparkSession,
     fileIndex: PartitioningAwareFileIndex,
     dataSchema: StructType)
-  extends ScanBuilder
+    extends ScanBuilder
     with SupportsPushDownRequiredColumns
     with SupportsPushDownCatalystFilters {
   private val partitionSchema = fileIndex.partitionSchema
@@ -73,10 +73,11 @@ abstract class FileScanBuilder(
     val (deterministicFilters, nonDeterminsticFilters) = 
filters.partition(_.deterministic)
     val (partitionFilters, dataFilters) =
       DataSourceUtils.getPartitionFiltersAndDataFilters(partitionSchema, 
deterministicFilters)
-    this.partitionFilters = partitionFilters.filter { f =>
+    val (partitionFiltersToPush, postScanPartitionFilters) = 
partitionFilters.partition { f =>
       // Python UDFs might exist because this rule is applied before 
``ExtractPythonUDFs``.
       !SubqueryExpression.hasSubquery(f) && 
!f.exists(_.isInstanceOf[PythonUDF])
     }
+    this.partitionFilters = partitionFiltersToPush
     this.dataFilters = dataFilters
     val translatedFilters = mutable.ArrayBuffer.empty[sources.Filter]
     for (filterExpr <- dataFilters) {
@@ -86,7 +87,7 @@ abstract class FileScanBuilder(
       }
     }
     pushedDataFilters = pushDataFilters(translatedFilters.toArray)
-    dataFilters ++ nonDeterminsticFilters
+    dataFilters ++ postScanPartitionFilters ++ nonDeterminsticFilters
   }
 
   override def pushedFilters: Array[Predicate] = pushedDataFilters.map(_.toV2)


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to