anoopj commented on code in PR #3247:
URL: https://github.com/apache/iceberg-rust/pull/3247#discussion_r4105910525
##########
crates/iceberg/src/expr/visitors/page_index_evaluator.rs:
##########
@@ -339,46 +341,91 @@ impl<'a> PageIndexEvaluator<'a> {
)
})
.collect(),
- ColumnIndexMetaData::BYTE_ARRAY(idx) => idx
- .min_values_iter()
- .zip(idx.max_values_iter())
- .enumerate()
- .zip(row_counts.iter())
- .map(|((i, (min, max)), &row_count)| {
- predicate(
- min.map(|val| {
- Datum::new(
- field_type.clone(),
-
PrimitiveLiteral::String(String::from_utf8(val.to_vec()).unwrap()),
- )
- }),
- max.map(|val| {
- Datum::new(
- field_type.clone(),
-
PrimitiveLiteral::String(String::from_utf8(val.to_vec()).unwrap()),
- )
- }),
+ ColumnIndexMetaData::BYTE_ARRAY(idx) => {
+ // Parquet stores Iceberg string and binary bounds as
BYTE_ARRAY.
+ // Any other field type on a BYTE_ARRAY column (e.g. a non-spec
+ // BYTE_ARRAY decimal) can't be decoded safely, so skip page
+ // pruning for it rather than risk pruning pages that match.
+ if !matches!(field_type, PrimitiveType::String |
PrimitiveType::Binary) {
+ tracing::debug!(
+ field_id,
+ %field_type,
+ "Skipping page-index pruning: BYTE_ARRAY column index
on a non-string/binary field"
+ );
+ return Ok(None);
+ }
+
+ let mut page_filter = Vec::with_capacity(row_counts.len());
+ for ((i, (min, max)), &row_count) in idx
+ .min_values_iter()
+ .zip(idx.max_values_iter())
+ .enumerate()
+ .zip(row_counts.iter())
+ {
+ // A bound that won't decode (e.g. a min/max stat truncated
+ // mid-UTF-8-sequence) means this column's page index can't
+ // be trusted, so skip pruning for the whole column rather
+ // than abort the scan.
+ let (Ok(min), Ok(max)) = (
+ min.map(|val|
Self::byte_array_bound_to_datum(field_type, val))
+ .transpose(),
+ max.map(|val|
Self::byte_array_bound_to_datum(field_type, val))
+ .transpose(),
+ ) else {
+ tracing::debug!(
Review Comment:
Good callout. The skip log now carries page_index and the decode error.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
