hakunamatata-sb commented on code in PR #784:
URL: https://github.com/apache/hudi-rs/pull/784#discussion_r4024082488
##########
crates/core/src/table/partition.rs:
##########
@@ -174,6 +174,71 @@ impl PartitionPruner {
})
}
+ /// Returns `true` if a partition directory prefix should be descended
into.
+ ///
+ /// Unlike `should_include`, `partial_path` may resolve fewer segments than
+ /// the schema has fields (we're mid-descent, not at a leaf yet). Any
segment
+ /// this can't safely reason about — not enough schema fields to compare
+ /// against, a single opaque path field (e.g. under a timestamp-based key
+ /// generator, where a segment carries no independently meaningful value),
+ /// a hive-style segment that doesn't parse as `key=value`, or a
cast/compare
+ /// error — is treated as unconstrained, never as a reason to reject. Only
a
+ /// segment that positively fails an already-resolved filter returns
`false`.
+ pub fn should_include_prefix(&self, partial_path: &str) -> bool {
+ if self.and_filters.is_empty() || self.schema.fields().len() <= 1 {
Review Comment:
fixed.
##########
crates/core/src/table/partition.rs:
##########
@@ -174,6 +174,71 @@ impl PartitionPruner {
})
}
+ /// Returns `true` if a partition directory prefix should be descended
into.
+ ///
+ /// Unlike `should_include`, `partial_path` may resolve fewer segments than
+ /// the schema has fields (we're mid-descent, not at a leaf yet). Any
segment
+ /// this can't safely reason about — not enough schema fields to compare
+ /// against, a single opaque path field (e.g. under a timestamp-based key
+ /// generator, where a segment carries no independently meaningful value),
+ /// a hive-style segment that doesn't parse as `key=value`, or a
cast/compare
+ /// error — is treated as unconstrained, never as a reason to reject. Only
a
+ /// segment that positively fails an already-resolved filter returns
`false`.
+ pub fn should_include_prefix(&self, partial_path: &str) -> bool {
+ if self.and_filters.is_empty() || self.schema.fields().len() <= 1 {
+ return true;
+ }
+
+ let decoded;
Review Comment:
fixed
##########
crates/core/src/table/partition.rs:
##########
@@ -401,6 +466,113 @@ mod tests {
assert!(!pruner.should_include("date=2023-02-01/category=B/count=10"));
}
+ #[test]
+ fn test_partition_pruner_should_include_prefix() {
+ let schema = create_test_schema();
+ let configs = create_hudi_configs(true, false);
+
+ let filter_gt_date = Filter::try_from(("date", ">",
"2023-01-01")).unwrap();
+ let filter_eq_a = Filter::try_from(("category", "=", "A")).unwrap();
+ let filter_lte_100 = Filter::try_from(("count", "<=", "100")).unwrap();
+
+ let pruner = PartitionPruner::new(
+ &[filter_gt_date, filter_eq_a, filter_lte_100],
+ &schema,
+ &configs,
+ )
+ .unwrap();
+
+ // A resolved segment that already violates its filter is rejected
+ // without needing to resolve the rest of the path.
+ assert!(!pruner.should_include_prefix("date=2022-12-31"));
+ assert!(!pruner.should_include_prefix("date=2023-02-01/category=B"));
+
+ // A resolved segment that satisfies its filter, with later fields not
+ // yet resolved, is not rejected.
+ assert!(pruner.should_include_prefix("date=2023-02-01"));
+ assert!(pruner.should_include_prefix("date=2023-02-01/category=A"));
+
assert!(pruner.should_include_prefix("date=2023-02-01/category=A/count=10"));
+ }
+
+ #[test]
+ fn test_partition_pruner_should_include_prefix_no_filters() {
+ let pruner = PartitionPruner::empty();
+ assert!(pruner.should_include_prefix("date=2022-12-31"));
+ assert!(pruner.should_include_prefix(""));
+ }
+
+ #[test]
+ fn
test_partition_pruner_should_include_prefix_malformed_segment_fails_open() {
+ let schema = create_test_schema();
+ let configs = create_hudi_configs(true, false);
+ let filter_eq_a = Filter::try_from(("category", "=", "A")).unwrap();
+ let pruner = PartitionPruner::new(&[filter_eq_a], &schema,
&configs).unwrap();
+
+ // A segment that doesn't parse as `key=value` under hive-style
+ // partitioning can't be safely compared, so it must not be rejected.
+ assert!(pruner.should_include_prefix("not-a-kv-pair"));
+ }
+
+ #[test]
+ fn test_partition_pruner_should_include_prefix_non_hive_style() {
+ let schema = create_test_schema();
+ let configs = create_hudi_configs(false, false);
+ let filter_gt_date = Filter::try_from(("date", ">",
"2023-01-01")).unwrap();
+ let pruner = PartitionPruner::new(&[filter_gt_date], &schema,
&configs).unwrap();
+
+ assert!(!pruner.should_include_prefix("2022-12-31"));
+ assert!(pruner.should_include_prefix("2023-02-01"));
+ }
+
+ #[test]
+ fn test_partition_pruner_should_include_prefix_url_encoded() {
+ let schema = create_test_schema();
+ let configs = create_hudi_configs(true, true);
+ let filter_eq_a = Filter::try_from(("category", "=", "A")).unwrap();
+ let pruner = PartitionPruner::new(&[filter_eq_a], &schema,
&configs).unwrap();
+
+
assert!(!pruner.should_include_prefix("date%3D2023-02-01%2Fcategory%3DB"));
Review Comment:
fixed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
