This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hudi-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 17149af4 fix(core): prevent incorrect timestamp partition pruning for
negative filters (#966)
17149af4 is described below
commit 17149af4ac5c2ecc21f81414be4e0369e07e9970
Author: slfan1989 <[email protected]>
AuthorDate: Sun Nov 8 14:22:42 2026 +0800
fix(core): prevent incorrect timestamp partition pruning for negative
filters (#966)
Skip timestamp partition pruning for `!=` and `NOT IN` filters to avoid
dropping matching rows. These predicates remain applied during row-level
filtering.
---
crates/core/src/keygen/timestamp_based.rs | 18 ++++-----
crates/core/src/table/partition.rs | 62 ++++++++++++++++++++++++++++++-
crates/core/tests/table_read_tests.rs | 30 +++++++++++++++
3 files changed, 98 insertions(+), 12 deletions(-)
diff --git a/crates/core/src/keygen/timestamp_based.rs
b/crates/core/src/keygen/timestamp_based.rs
index 5d124ffc..36fad316 100644
--- a/crates/core/src/keygen/timestamp_based.rs
+++ b/crates/core/src/keygen/timestamp_based.rs
@@ -509,7 +509,9 @@ impl KeyGeneratorFilterTransformer for
TimestampBasedKeyGenerator {
let field = MetaField::PartitionPath.as_ref().to_string();
match filter.operator {
- ExprOperator::Eq | ExprOperator::Ne => {
+ // Many timestamps share one partition, so excluding a timestamp
cannot exclude it.
+ ExprOperator::Ne | ExprOperator::NotIn => Ok(vec![]),
+ ExprOperator::Eq => {
let dt = self.parse_timestamp(&filter.values[0])?;
let path = self.format_partition_path(&dt);
Ok(vec![Filter {
@@ -518,7 +520,7 @@ impl KeyGeneratorFilterTransformer for
TimestampBasedKeyGenerator {
values: vec![path],
}])
}
- ExprOperator::In | ExprOperator::NotIn => {
+ ExprOperator::In => {
let paths: Result<Vec<String>> = filter
.values
.iter()
@@ -909,7 +911,7 @@ mod tests {
assert_eq!(transformed.len(), 1);
assert_eq!(transformed[0].values[0], "2024/01/25");
- // Not-equal: now supported
+ // Negative predicates must be evaluated against the original
timestamps.
let keygen =
TimestampBasedKeyGenerator::from_configs(&create_test_configs_date_string()).unwrap();
let filter = Filter {
@@ -918,12 +920,7 @@ mod tests {
values: vec!["2023-04-01T12:01:00.123Z".to_string()],
};
let transformed = keygen.transform_filter(&filter).unwrap();
- assert_eq!(transformed.len(), 1);
- assert_eq!(transformed[0].operator, ExprOperator::Ne);
- assert_eq!(
- transformed[0].values[0],
- "year=2023/month=04/day=01/hour=12"
- );
+ assert!(transformed.is_empty());
// Range operators: Gt/Gte → Gte, Lt/Lte → Lte (safe widening)
let keygen =
@@ -1008,8 +1005,7 @@ mod tests {
)
.unwrap();
let transformed = keygen.transform_filter(&filter).unwrap();
- assert_eq!(transformed.len(), 1);
- assert_eq!(transformed[0].operator, ExprOperator::NotIn);
+ assert!(transformed.is_empty());
}
#[test]
diff --git a/crates/core/src/table/partition.rs
b/crates/core/src/table/partition.rs
index 1539658d..9d564c36 100644
--- a/crates/core/src/table/partition.rs
+++ b/crates/core/src/table/partition.rs
@@ -307,7 +307,7 @@ mod tests {
use crate::expr::ExprOperator;
use arrow::datatypes::{DataType, Field, Schema};
- use arrow_array::Date32Array;
+ use arrow_array::{BooleanArray, Date32Array, Int64Array, RecordBatch};
use std::str::FromStr;
fn create_test_schema() -> Schema {
@@ -704,6 +704,66 @@ mod tests {
assert_eq!(transformed[0].values[0], user_filter.values[0]);
}
+ #[test]
+ fn test_timestamp_negative_filters_keep_matching_rows() {
+ // Excluding midnight must not hide the noon record in the same
partition.
+ let batch = RecordBatch::try_from_iter([(
+ "ts",
+ Arc::new(Int64Array::from(vec![
+ 1706140800, 1706184000, 1706227200, 1706270400,
+ ])) as ArrayRef,
+ )])
+ .unwrap();
+ let partition_schema = Schema::new(vec![Field::new(
+ MetaField::PartitionPath.as_ref(),
+ DataType::Utf8,
+ false,
+ )]);
+ let configs = HudiConfigs::new([
+ ("hoodie.table.partition.fields", "ts"),
+ (
+ "hoodie.table.keygenerator.class",
+ "org.apache.hudi.keygen.TimestampBasedKeyGenerator",
+ ),
+ ("hoodie.keygen.timebased.timestamp.type", "UNIX_TIMESTAMP"),
+ ("hoodie.keygen.timebased.output.dateformat", "yyyy/MM/dd"),
+ ]);
+ let paths = ["2024/01/25", "2024/01/26"];
+ for (operator, excluded, expected) in [
+ (
+ ExprOperator::Ne,
+ vec!["1706140800"],
+ vec![false, true, true, true],
+ ),
+ (
+ ExprOperator::NotIn,
+ vec!["1706140800", "1706227200"],
+ vec![false, true, false, true],
+ ),
+ ] {
+ let filter = Filter::new(
+ "ts".to_string(),
+ operator,
+ excluded.into_iter().map(str::to_string).collect(),
+ )
+ .unwrap();
+ let filters = std::slice::from_ref(&filter);
+ let pruner = PartitionPruner::new(filters, &partition_schema,
&configs).unwrap();
+ for path in paths {
+ assert!(pruner.should_include(path), "{operator} must retain
{path}");
+ }
+ assert_eq!(
+ crate::expr::filter::filters_to_row_mask(filters,
&batch).unwrap(),
+ BooleanArray::from(expected),
+ );
+ let bound = Filter::try_from(("ts", ">=", "1706227200")).unwrap();
+ let bounded =
+ PartitionPruner::new(&[filter, bound], &partition_schema,
&configs).unwrap();
+ assert!(!bounded.should_include(paths[0]));
+ assert!(bounded.should_include(paths[1]));
+ }
+ }
+
#[test]
fn test_partition_pruner_with_timestamp_keygen() {
let configs = HudiConfigs::new([
diff --git a/crates/core/tests/table_read_tests.rs
b/crates/core/tests/table_read_tests.rs
index 044b86c8..afced386 100644
--- a/crates/core/tests/table_read_tests.rs
+++ b/crates/core/tests/table_read_tests.rs
@@ -1116,6 +1116,36 @@ mod v9_tables {
Ok(())
}
+ #[tokio::test]
+ async fn
test_timebasedkeygen_unixtimestamp_cow_snapshot_with_negative_filters()
+ -> Result<()> {
+ let base_url =
SampleTable::V9TimebasedkeygenUnixtimestamp.url_to_cow();
+ let hudi_table = open_table(base_url.path()).await?;
+ let all_rows = read_txn_rows_from_snapshot(&hudi_table).await?;
+
+ // TXN-001 shares the 2024/01/15 partition with TXN-002, and
TXN-005 shares
+ // 2024/01/17 with TXN-006.
+ for (op, value, excluded) in [
+ ("!=", "1705311000", vec!["TXN-001"]),
+ (
+ "NOT IN",
+ "1705311000,1705485600",
+ vec!["TXN-001", "TXN-005"],
+ ),
+ ] {
+ let records = hudi_table
+ .read(&ReadOptions::new().with_filters([("ts_epoch", op,
value)])?)
+ .await?;
+ let expected: Vec<_> = all_rows
+ .iter()
+ .filter(|(txn_id, _, _)|
!excluded.contains(&txn_id.as_str()))
+ .cloned()
+ .collect();
+ assert_eq!(txn_rows(&records), expected, "ts_epoch {op}
{value}");
+ }
+ Ok(())
+ }
+
#[tokio::test]
async fn test_txns_simple_overwrite_cow_snapshot() -> Result<()> {
let base_url = SampleTable::V9TxnsSimpleOverwrite.url_to_cow();