zhuqi-lucas commented on code in PR #7513: URL: https://github.com/apache/arrow-rs/pull/7513#discussion_r2113966651
########## parquet/src/arrow/array_reader/incremental_batch_builder.rs: ########## @@ -0,0 +1,333 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::arrow::array_reader::incremental_array_builder::{ + GenericIncrementalArrayBuilder, IncrementalArrayBuilder, +}; +use arrow_array::builder::{BinaryViewBuilder, PrimitiveBuilder, StringViewBuilder}; +use arrow_array::{downcast_primitive, BooleanArray, RecordBatch}; +use arrow_schema::{ArrowError, DataType, SchemaRef}; +use arrow_select::filter::{FilterBuilder, FilterPredicate, SlicesIterator}; +use std::borrow::Cow; +use std::collections::VecDeque; + +type ArrayBuilderImpl = Box<dyn IncrementalArrayBuilder>; + +/// Incrementally creates `RecordBatch`es of limited size +/// +/// This structure implements the common pattern of incrementally creating +/// output batches of a specific size from an input stream of arrays. +/// +/// See: <https://github.com/apache/arrow-rs/issues/6692> +/// +/// This is a convenience over [`IncrementalArrayBuilder`] which is used to +/// build arrays for each column in the `RecordBatch`. +/// +/// Which rows are selected from the input arrays are be chosen using one of the +/// following mechanisms: +/// +/// 1. `concat`-enated: all rows from the input array are appended +/// 2. `filter`-ed: the input array is filtered using a `BooleanArray` +/// 3. `take`-n: a subset of the input array is selected based on the indices provided in a `UInt32Array` or similar. +/// +/// This structure handles multiple arrays +pub struct IncrementalRecordBatchBuilder { Review Comment: Great work, very nice to append filter to apply to batch! -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
