This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hudi-rs.git


The following commit(s) were added to refs/heads/main by this push:
     new 17149af4 fix(core): prevent incorrect timestamp partition pruning for 
negative filters (#966)
17149af4 is described below

commit 17149af4ac5c2ecc21f81414be4e0369e07e9970
Author: slfan1989 <[email protected]>
AuthorDate: Sun Nov 8 14:22:42 2026 +0800

    fix(core): prevent incorrect timestamp partition pruning for negative 
filters (#966)
    
    Skip timestamp partition pruning for `!=` and `NOT IN` filters to avoid 
dropping matching rows. These predicates remain applied during row-level 
filtering.
---
 crates/core/src/keygen/timestamp_based.rs | 18 ++++-----
 crates/core/src/table/partition.rs        | 62 ++++++++++++++++++++++++++++++-
 crates/core/tests/table_read_tests.rs     | 30 +++++++++++++++
 3 files changed, 98 insertions(+), 12 deletions(-)

diff --git a/crates/core/src/keygen/timestamp_based.rs 
b/crates/core/src/keygen/timestamp_based.rs
index 5d124ffc..36fad316 100644
--- a/crates/core/src/keygen/timestamp_based.rs
+++ b/crates/core/src/keygen/timestamp_based.rs
@@ -509,7 +509,9 @@ impl KeyGeneratorFilterTransformer for 
TimestampBasedKeyGenerator {
         let field = MetaField::PartitionPath.as_ref().to_string();
 
         match filter.operator {
-            ExprOperator::Eq | ExprOperator::Ne => {
+            // Many timestamps share one partition, so excluding a timestamp 
cannot exclude it.
+            ExprOperator::Ne | ExprOperator::NotIn => Ok(vec![]),
+            ExprOperator::Eq => {
                 let dt = self.parse_timestamp(&filter.values[0])?;
                 let path = self.format_partition_path(&dt);
                 Ok(vec![Filter {
@@ -518,7 +520,7 @@ impl KeyGeneratorFilterTransformer for 
TimestampBasedKeyGenerator {
                     values: vec![path],
                 }])
             }
-            ExprOperator::In | ExprOperator::NotIn => {
+            ExprOperator::In => {
                 let paths: Result<Vec<String>> = filter
                     .values
                     .iter()
@@ -909,7 +911,7 @@ mod tests {
         assert_eq!(transformed.len(), 1);
         assert_eq!(transformed[0].values[0], "2024/01/25");
 
-        // Not-equal: now supported
+        // Negative predicates must be evaluated against the original 
timestamps.
         let keygen =
             
TimestampBasedKeyGenerator::from_configs(&create_test_configs_date_string()).unwrap();
         let filter = Filter {
@@ -918,12 +920,7 @@ mod tests {
             values: vec!["2023-04-01T12:01:00.123Z".to_string()],
         };
         let transformed = keygen.transform_filter(&filter).unwrap();
-        assert_eq!(transformed.len(), 1);
-        assert_eq!(transformed[0].operator, ExprOperator::Ne);
-        assert_eq!(
-            transformed[0].values[0],
-            "year=2023/month=04/day=01/hour=12"
-        );
+        assert!(transformed.is_empty());
 
         // Range operators: Gt/Gte → Gte, Lt/Lte → Lte (safe widening)
         let keygen =
@@ -1008,8 +1005,7 @@ mod tests {
         )
         .unwrap();
         let transformed = keygen.transform_filter(&filter).unwrap();
-        assert_eq!(transformed.len(), 1);
-        assert_eq!(transformed[0].operator, ExprOperator::NotIn);
+        assert!(transformed.is_empty());
     }
 
     #[test]
diff --git a/crates/core/src/table/partition.rs 
b/crates/core/src/table/partition.rs
index 1539658d..9d564c36 100644
--- a/crates/core/src/table/partition.rs
+++ b/crates/core/src/table/partition.rs
@@ -307,7 +307,7 @@ mod tests {
     use crate::expr::ExprOperator;
 
     use arrow::datatypes::{DataType, Field, Schema};
-    use arrow_array::Date32Array;
+    use arrow_array::{BooleanArray, Date32Array, Int64Array, RecordBatch};
     use std::str::FromStr;
 
     fn create_test_schema() -> Schema {
@@ -704,6 +704,66 @@ mod tests {
         assert_eq!(transformed[0].values[0], user_filter.values[0]);
     }
 
+    #[test]
+    fn test_timestamp_negative_filters_keep_matching_rows() {
+        // Excluding midnight must not hide the noon record in the same 
partition.
+        let batch = RecordBatch::try_from_iter([(
+            "ts",
+            Arc::new(Int64Array::from(vec![
+                1706140800, 1706184000, 1706227200, 1706270400,
+            ])) as ArrayRef,
+        )])
+        .unwrap();
+        let partition_schema = Schema::new(vec![Field::new(
+            MetaField::PartitionPath.as_ref(),
+            DataType::Utf8,
+            false,
+        )]);
+        let configs = HudiConfigs::new([
+            ("hoodie.table.partition.fields", "ts"),
+            (
+                "hoodie.table.keygenerator.class",
+                "org.apache.hudi.keygen.TimestampBasedKeyGenerator",
+            ),
+            ("hoodie.keygen.timebased.timestamp.type", "UNIX_TIMESTAMP"),
+            ("hoodie.keygen.timebased.output.dateformat", "yyyy/MM/dd"),
+        ]);
+        let paths = ["2024/01/25", "2024/01/26"];
+        for (operator, excluded, expected) in [
+            (
+                ExprOperator::Ne,
+                vec!["1706140800"],
+                vec![false, true, true, true],
+            ),
+            (
+                ExprOperator::NotIn,
+                vec!["1706140800", "1706227200"],
+                vec![false, true, false, true],
+            ),
+        ] {
+            let filter = Filter::new(
+                "ts".to_string(),
+                operator,
+                excluded.into_iter().map(str::to_string).collect(),
+            )
+            .unwrap();
+            let filters = std::slice::from_ref(&filter);
+            let pruner = PartitionPruner::new(filters, &partition_schema, 
&configs).unwrap();
+            for path in paths {
+                assert!(pruner.should_include(path), "{operator} must retain 
{path}");
+            }
+            assert_eq!(
+                crate::expr::filter::filters_to_row_mask(filters, 
&batch).unwrap(),
+                BooleanArray::from(expected),
+            );
+            let bound = Filter::try_from(("ts", ">=", "1706227200")).unwrap();
+            let bounded =
+                PartitionPruner::new(&[filter, bound], &partition_schema, 
&configs).unwrap();
+            assert!(!bounded.should_include(paths[0]));
+            assert!(bounded.should_include(paths[1]));
+        }
+    }
+
     #[test]
     fn test_partition_pruner_with_timestamp_keygen() {
         let configs = HudiConfigs::new([
diff --git a/crates/core/tests/table_read_tests.rs 
b/crates/core/tests/table_read_tests.rs
index 044b86c8..afced386 100644
--- a/crates/core/tests/table_read_tests.rs
+++ b/crates/core/tests/table_read_tests.rs
@@ -1116,6 +1116,36 @@ mod v9_tables {
             Ok(())
         }
 
+        #[tokio::test]
+        async fn 
test_timebasedkeygen_unixtimestamp_cow_snapshot_with_negative_filters()
+        -> Result<()> {
+            let base_url = 
SampleTable::V9TimebasedkeygenUnixtimestamp.url_to_cow();
+            let hudi_table = open_table(base_url.path()).await?;
+            let all_rows = read_txn_rows_from_snapshot(&hudi_table).await?;
+
+            // TXN-001 shares the 2024/01/15 partition with TXN-002, and 
TXN-005 shares
+            // 2024/01/17 with TXN-006.
+            for (op, value, excluded) in [
+                ("!=", "1705311000", vec!["TXN-001"]),
+                (
+                    "NOT IN",
+                    "1705311000,1705485600",
+                    vec!["TXN-001", "TXN-005"],
+                ),
+            ] {
+                let records = hudi_table
+                    .read(&ReadOptions::new().with_filters([("ts_epoch", op, 
value)])?)
+                    .await?;
+                let expected: Vec<_> = all_rows
+                    .iter()
+                    .filter(|(txn_id, _, _)| 
!excluded.contains(&txn_id.as_str()))
+                    .cloned()
+                    .collect();
+                assert_eq!(txn_rows(&records), expected, "ts_epoch {op} 
{value}");
+            }
+            Ok(())
+        }
+
         #[tokio::test]
         async fn test_txns_simple_overwrite_cow_snapshot() -> Result<()> {
             let base_url = SampleTable::V9TxnsSimpleOverwrite.url_to_cow();

Reply via email to