mbutrovich commented on code in PR #3724:
URL: https://github.com/apache/iceberg-rust/pull/3724#discussion_r4621319435


##########
crates/iceberg/src/arrow/manifest_filter.rs:
##########
@@ -0,0 +1,1245 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Filtering manifests read into Arrow by a scan's row filter.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use arrow_arith::boolean::{and, is_null, not, or};
+use arrow_array::cast::AsArray;
+use arrow_array::types::Int32Type;
+use arrow_array::{
+    Array, ArrayRef, BinaryArray, BooleanArray, Decimal128Array, Float32Array, 
Float64Array,
+    Int32Array, Int64Array, RecordBatch, Scalar,
+};
+use arrow_buffer::{BooleanBuffer, BooleanBufferBuilder};
+use arrow_ord::cmp::{gt, gt_eq, lt, lt_eq};
+use arrow_schema::ArrowError;
+use arrow_select::take::take;
+use fnv::FnvHashSet;
+use typed_builder::TypedBuilder;
+
+use crate::error::invalid_data;
+use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, 
visit};
+use crate::expr::visitors::inclusive_projection::project_partition_filter;
+use crate::expr::visitors::manifest_evaluator::IN_PREDICATE_LIMIT;
+use crate::expr::{Bind, BoundPredicate, BoundReference, Predicate};
+use crate::spec::decimal_utils::i128_from_be_bytes;
+use crate::spec::{Datum, PartitionSpecRef, PrimitiveLiteral, PrimitiveType, 
SchemaRef};
+use crate::{Error, ErrorKind, Result};
+
+/// Evaluates a row filter against batches from
+/// [`read_manifest_list`](crate::arrow::read_manifest_list), with the same
+/// results as the manifest pruning in [`TableScan`](crate::scan::TableScan).
+///
+/// # Example
+///
+/// ```
+/// use std::sync::Arc;
+///
+/// use iceberg::arrow::{ManifestFilter, read_manifest_list};
+/// use iceberg::expr::Reference;
+/// use iceberg::spec::{Datum, NestedField, PartitionSpec, PrimitiveType, 
Schema, Type};
+///
+/// let schema = Arc::new(
+///     Schema::builder()
+///         .with_fields(vec![
+///             NestedField::required(1, "id", 
Type::Primitive(PrimitiveType::Long)).into(),
+///         ])
+///         .build()?,
+/// );
+/// let filter = ManifestFilter::builder()
+///     .schema(schema.clone())
+///     .partition_specs(vec![Arc::new(PartitionSpec::unpartition_spec())])
+///     .filter(Reference::new("id").equal_to(Datum::long(1)))
+///     .build()?;
+///
+/// let bytes = std::fs::read(concat!(
+///     env!("CARGO_MANIFEST_DIR"),
+///     "/testdata/manifests_lists/manifest-list-v2-2.avro"
+/// ))?;
+/// for batch in read_manifest_list(&bytes)? {
+///     // An unpartitioned manifest may contain any row.
+///     let mask = filter.manifest_list_mask(&batch?)?;
+///     assert!(mask.values().iter().all(|might_match| might_match));
+/// }
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
+#[derive(Debug, TypedBuilder)]
+#[builder(build_method(into = Result<ManifestFilter>))]
+pub struct ManifestFilter {
+    schema: SchemaRef,
+    partition_specs: Vec<PartitionSpecRef>,
+    filter: Predicate,
+    #[builder(default = true)]
+    case_sensitive: bool,
+    /// The filter projected to each spec's partition values, by spec ID.
+    #[builder(setter(skip), default)]
+    partition_filters: HashMap<i32, PartitionFilter>,
+}
+
+#[derive(Debug)]
+struct PartitionFilter {
+    field_count: usize,
+    filter: BoundPredicate,
+}
+
+impl From<ManifestFilter> for Result<ManifestFilter> {
+    fn from(mut manifest_filter: ManifestFilter) -> Self {
+        // `TableScanBuilder::with_filter` rewrites NOT the same way, and the
+        // projection rejects NOT.
+        let filter = manifest_filter.filter.clone().rewrite_not().bind(
+            manifest_filter.schema.clone(),
+            manifest_filter.case_sensitive,
+        )?;
+        for spec in &manifest_filter.partition_specs {
+            let partition_filter = PartitionFilter {
+                field_count: spec.fields().len(),
+                // A spec that references a dropped source column can't prune, 
as
+                // in `TableScan`.
+                filter: project_partition_filter(
+                    &manifest_filter.schema,
+                    spec,
+                    &filter,
+                    manifest_filter.case_sensitive,
+                )?
+                .unwrap_or(BoundPredicate::AlwaysTrue),
+            };
+            let previous = manifest_filter
+                .partition_filters
+                .insert(spec.spec_id(), partition_filter);
+            if previous.is_some() {
+                return Err(invalid_data!(
+                    "duplicate partition spec ID {}",
+                    spec.spec_id()
+                ));
+            }
+        }
+        Ok(manifest_filter)
+    }
+}
+
+impl ManifestFilter {
+    /// Returns true for each manifest in `batch` that may contain rows 
matching
+    /// the filter, judged by its partition summaries.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if `batch` doesn't have the schema that
+    /// [`read_manifest_list`](crate::arrow::read_manifest_list) returns, if a
+    /// manifest's partition spec isn't one the filter was built with, or if 
its
+    /// partition summaries don't match the spec.
+    pub fn manifest_list_mask(&self, batch: &RecordBatch) -> 
Result<BooleanArray> {
+        let spec_ids = column(batch, "partition_spec_id")?
+            .as_primitive_opt::<Int32Type>()
+            .ok_or_else(|| invalid_data!("partition_spec_id is not an int 
column"))?;
+        let partitions = column(batch, "partitions")?
+            .as_list_opt::<i32>()
+            .ok_or_else(|| invalid_data!("partitions is not a list column"))?;
+        let summaries = SummaryColumns::try_new(partitions.values())?;
+
+        // A manifest without partition summaries may contain any row, as in
+        // `ManifestEvaluator`.
+        let mut rows_by_spec: HashMap<i32, Vec<usize>> = HashMap::new();
+        for row in 0..batch.num_rows() {
+            if partitions.is_valid(row) && partitions.value_length(row) > 0 {
+                if spec_ids.is_null(row) {
+                    return Err(invalid_data!("manifest has no 
partition_spec_id"));
+                }
+                rows_by_spec
+                    .entry(spec_ids.value(row))
+                    .or_default()
+                    .push(row);
+            }
+        }
+
+        let mut mask = BooleanBufferBuilder::new(batch.num_rows());
+        mask.append_n(batch.num_rows(), true);
+        for (spec_id, rows) in rows_by_spec {
+            let partition_filter = self
+                .partition_filters
+                .get(&spec_id)
+                .ok_or_else(|| invalid_data!("no partition spec with ID 
{spec_id}"))?;
+            let mut starts = Vec::with_capacity(rows.len());
+            for &row in &rows {
+                let length = usize::try_from(partitions.value_length(row))?;

Review Comment:
   This checks the number of summaries even when the spec's filter projects to 
`AlwaysTrue` and nothing can be pruned. `ManifestEvaluator` doesn't check it 
and [indexes the summaries by 
position](https://github.com/apache/iceberg-rust/blob/641f59049e0aefad0c60e19915ecdb54933e002f/crates/iceberg/src/expr/visitors/manifest_evaluator.rs#L443-L446).
 A manifest list whose summaries don't match their spec is corrupt, so this 
returns an error whatever the filter is.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to