This is an automated email from the ASF dual-hosted git repository.

voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 8e5608ecaf6e fix(spark): skip partition stats pruning for time travel 
queries (#20451)
8e5608ecaf6e is described below

commit 8e5608ecaf6e3f78b33d83ac0bd4e4712cb81bd6
Author: Shuo Cheng <[email protected]>
AuthorDate: Wed Nov 4 23:59:13 2026 +0800

    fix(spark): skip partition stats pruning for time travel queries (#20451)
---
 .../scala/org/apache/hudi/HoodieFileIndex.scala    |  9 ++--
 .../apache/hudi/PartitionStatsIndexSupport.scala   | 13 +++++
 .../hudi/functional/TestTimeTravelQuery.scala      | 62 +++++++++++++++++++++-
 3 files changed, 77 insertions(+), 7 deletions(-)

diff --git 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala
 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala
index eafff4e8ddb8..c0be0c49b33b 100644
--- 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala
+++ 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala
@@ -341,6 +341,7 @@ case class HoodieFileIndex(spark: SparkSession,
                                       partitionFilters: Seq[Expression]):
   (Boolean, Seq[(Option[BaseHoodieTableFileIndex.PartitionPath], 
Seq[FileSlice])]) = {
     val isPartitionedTable = getPartitionColumns.length > 0
+    lazy val partitionStatsIndexSupport = new 
PartitionStatsIndexSupport(spark, schema, rawHoodieSchema, metadataConfig, 
metaClient)
     val prunedPartitionsTuple: (Boolean, Seq[PartitionPath]) =
       if (isPartitionedTable && partitionFilters.nonEmpty) {
         // For partitioned table and partition filters, prune the partitions 
by the partition filters
@@ -349,11 +350,9 @@ case class HoodieFileIndex(spark: SparkSession,
         } else {
           (true, listMatchingPartitionPaths(partitionFilters))
         }
-      } else if (isPartitionedTable && isDataSkippingEnabled) {
-        // For partitioned table and no partition filters, if data skipping is 
enabled,
-        // try using the PARTITION_STATS index to prune the partitions
-        val prunedPartitionPaths = new PartitionStatsIndexSupport(spark, 
schema, rawHoodieSchema, metadataConfig, metaClient)
-          .prunePartitions(this, dataFilters)
+      } else if (isPartitionedTable && isDataSkippingEnabled && 
partitionStatsIndexSupport.supportsQueryType(options)) {
+        // Both PARTITION_STATS and the expression index fallback use 
latest-snapshot partition statistics.
+        val prunedPartitionPaths = 
partitionStatsIndexSupport.prunePartitions(this, dataFilters)
         if (prunedPartitionPaths.nonEmpty) {
           try {
             (true, prunedPartitionPaths.get.map(e => 
convertToPartitionPath(e)).toSeq)
diff --git 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/PartitionStatsIndexSupport.scala
 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/PartitionStatsIndexSupport.scala
index 82a8f15f3663..d738fde19a2e 100644
--- 
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/PartitionStatsIndexSupport.scala
+++ 
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/PartitionStatsIndexSupport.scala
@@ -19,6 +19,7 @@
 
 package org.apache.hudi
 
+import org.apache.hudi.DataSourceReadOptions.TIME_TRAVEL_AS_OF_INSTANT
 import org.apache.hudi.HoodieConversionUtils.toScalaOption
 import org.apache.hudi.HoodieFileIndex.collectReferencedColumns
 import org.apache.hudi.avro.model.{HoodieMetadataColumnStats, 
HoodieMetadataRecord}
@@ -27,6 +28,7 @@ import org.apache.hudi.common.data.{HoodieData, 
HoodieListData}
 import org.apache.hudi.common.model.{FileSlice, HoodieRecord}
 import org.apache.hudi.common.schema.HoodieSchema
 import org.apache.hudi.common.table.HoodieTableMetaClient
+import 
org.apache.hudi.common.table.timeline.InstantComparison.{compareTimestamps, 
GREATER_THAN_OR_EQUALS}
 import org.apache.hudi.common.util.ValidationUtils.checkState
 import org.apache.hudi.core.read.BaseHoodieTableFileIndex
 import org.apache.hudi.metadata.{ColumnStatsIndexPrefixRawKey, 
HoodieMetadataPayload, HoodieTableMetadataUtil}
@@ -38,6 +40,7 @@ import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.expressions.{And, DateAdd, 
DateFormatClass, DateSub, Expression, FromUnixTime, ParseToDate, 
ParseToTimestamp, RegExpExtract, RegExpReplace, StringSplit, StringTrim, 
StringTrimLeft, StringTrimRight, Substring, UnaryExpression, UnixTimestamp}
 import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral
 import 
org.apache.spark.sql.hudi.DataSkippingUtils.{containsNullOrValueCountBasedFilters,
 translateIntoColumnStatsIndexFilterExpr}
+import org.apache.spark.sql.hudi.HoodieSqlCommonUtils
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.storage.StorageLevel
 
@@ -59,6 +62,16 @@ class PartitionStatsIndexSupport(spark: SparkSession,
       metaClient.getIndexMetadata.isPresent
   }
 
+  override def supportsQueryType(options: Map[String, String]): Boolean = {
+    // Partition statistics describe the latest snapshot. Their bounds can 
shrink after updates
+    // or partition replacement and exclude historical records.
+    options.get(TIME_TRAVEL_AS_OF_INSTANT.key).forall { instant =>
+      val latestCommit = 
metaClient.getCommitsTimeline.filterCompletedInstants.lastInstant
+      latestCommit.isPresent && 
compareTimestamps(HoodieSqlCommonUtils.formatQueryInstant(instant),
+        GREATER_THAN_OR_EQUALS, latestCommit.get.requestedTime)
+    }
+  }
+
   override def computeCandidateFileNames(fileIndex: HoodieFileIndex,
                                          queryFilters: Seq[Expression],
                                          queryReferencedColumns: Seq[String],
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala
index 4593e21839a7..aefc02f072be 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala
@@ -17,11 +17,11 @@
 
 package org.apache.hudi.functional
 
-import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, 
ScalaAssertionSupport}
+import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, 
HoodieFileIndex, ScalaAssertionSupport}
 import org.apache.hudi.common.config.HoodieMetadataConfig
 import org.apache.hudi.common.model.{HoodieCleaningPolicy, HoodieTableType}
 import org.apache.hudi.common.model.HoodieTableType.{COPY_ON_WRITE, 
MERGE_ON_READ}
-import org.apache.hudi.common.table.{HoodieTableConfig, TableSchemaResolver}
+import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, 
TableSchemaResolver}
 import org.apache.hudi.common.table.timeline.TimelineUtils
 import org.apache.hudi.common.testutils.{HoodieTestTable, HoodieTestUtils}
 import org.apache.hudi.config.{HoodieArchivalConfig, HoodieCleanConfig, 
HoodieCompactionConfig, HoodieWriteConfig}
@@ -30,6 +30,8 @@ import org.apache.hudi.testutils.HoodieSparkClientTestBase
 
 import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession}
 import org.apache.spark.sql.SaveMode.{Append, Overwrite}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, 
Literal}
+import org.apache.spark.sql.types.IntegerType
 import org.junit.jupiter.api.{AfterEach, BeforeEach}
 import org.junit.jupiter.api.Assertions.{assertEquals, assertNotNull, 
assertNull, assertTrue}
 import org.junit.jupiter.params.ParameterizedTest
@@ -226,6 +228,62 @@ class TestTimeTravelQuery extends 
HoodieSparkClientTestBase with ScalaAssertionS
     assertTrue(result4.isEmpty)
   }
 
+  @ParameterizedTest
+  @EnumSource(value = classOf[HoodieTableType])
+  def testTimeTravelWithPartitionStats(tableType: HoodieTableType): Unit = {
+    initMetaClient(tableType)
+    val _spark = spark
+    import _spark.implicits._
+
+    val opts = commonOpts ++ Map(
+      DataSourceWriteOptions.TABLE_TYPE.key -> tableType.name,
+      DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "dt",
+      HoodieMetadataConfig.ENABLE.key -> "true",
+      HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "true",
+      HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> "true",
+      HoodieMetadataConfig.COLUMN_STATS_INDEX_FOR_COLUMNS.key -> "value",
+      HoodieCompactionConfig.INLINE_COMPACT.key -> (tableType == 
MERGE_ON_READ).toString,
+      HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS.key -> "1"
+    )
+    val firstCommit = writeBatch(
+      Seq((1, 10, 1000, "p1"), (2, 100, 1000, "p2")).toDF("id", "value", 
"version", "dt"), opts, Overwrite)
+    // Update record id=1 from 10 to 50. The two snapshots contain:
+    //                  partition dt='p1'          partition dt='p2'
+    //   firstCommit:   (id=1, value=10)            (id=2, value=100)
+    //   latestCommit:  (id=1, value=50)            (id=2, value=100)
+    // COW rewrites the base file; MOR compacts inline. The latest min/max for 
column 'value'
+    // are therefore [50, 50] in partition dt='p1' and [100, 100] in partition 
dt='p2'.
+    val latestCommit = writeBatch(
+      Seq((1, 50, 1001, "p1")).toDF("id", "value", "version", "dt"), opts)
+
+    val readOpts = Map(
+      "path" -> basePath,
+      HoodieMetadataConfig.ENABLE.key -> "true",
+      DataSourceReadOptions.ENABLE_DATA_SKIPPING.key -> "true")
+    val valueFilter = EqualTo(AttributeReference("value", IntegerType)(), 
Literal(10))
+
+    metaClient = HoodieTableMetaClient.reload(metaClient)
+    // Query the latest snapshot WHERE value = 10: neither record matches, so 
statistics should
+    // prune both partitions, with as.of.instant either omitted or set to 
latestCommit.
+    Seq(None, Some(latestCommit)).foreach { instant =>
+      val options = readOpts ++ 
instant.map(DataSourceReadOptions.TIME_TRAVEL_AS_OF_INSTANT.key -> _)
+      val fileIndex = HoodieFileIndex(spark, metaClient, None, options)
+      val (pruned, partitions) = 
fileIndex.prunePartitionsAndGetFileSlices(Seq(valueFilter), Seq.empty)
+      assertTrue(pruned)
+      assertTrue(partitions.isEmpty, s"Unexpected partitions: $partitions")
+    }
+
+    // Query AS OF firstCommit WHERE value = 10: expect (id=1, value=10, 
version=1000, dt='p1').
+    // Applying the latest [50, 50] bounds would skip partition dt='p1' and 
incorrectly return no rows.
+    // Check both the commit instant string and its formatted date-time 
representation.
+    Seq(firstCommit, defaultDateTimeFormat(firstCommit)).foreach { instant =>
+      val options = readOpts + 
(DataSourceReadOptions.TIME_TRAVEL_AS_OF_INSTANT.key -> instant)
+      val result = spark.read.format("hudi").options(options).load()
+        .filter("value = 10").select("id", "value", "version", 
"dt").collect().toSeq
+      assertEquals(Seq(Row(1, 10, 1000, "p1")), result)
+    }
+  }
+
   private def writeBatch(df: DataFrame, options: Map[String, String], mode: 
SaveMode = Append): String = {
     df.write.format("hudi").options(options).mode(mode).save(basePath)
     metaClient.reloadActiveTimeline()

Reply via email to