alexanderbianchi commented on code in PR #3514:
URL: https://github.com/apache/iceberg-rust/pull/3514#discussion_r4407297345
##########
crates/iceberg/src/arrow/reader/options.rs:
##########
@@ -15,70 +15,324 @@
// specific language governing permissions and limitations
// under the License.
-//! Tunables for Parquet file I/O used by `ArrowReader`.
-
+use serde::{Deserialize, Serialize};
use typed_builder::TypedBuilder;
use super::{
DEFAULT_METADATA_SIZE_HINT, DEFAULT_RANGE_COALESCE_BYTES,
DEFAULT_RANGE_FETCH_CONCURRENCY,
};
+/// Configures Arrow reads independently of file planning and local I/O
resources.
+///
+/// ```no_run
+/// # use iceberg::{Result, arrow::{ArrowReaderOptions, ParquetReadOptions},
table::Table};
+/// # async fn read(table: &Table) -> Result<()> {
+/// let options = ArrowReaderOptions::builder()
+/// .with_batch_size(Some(1024))
+/// .with_parquet_options(
+/// ParquetReadOptions::builder()
+/// .with_metadata_size_hint(512 * 1024)
+/// .build(),
+/// )
+/// .build();
+/// let stream = table.scan().build()?.to_arrow(options).await?;
+/// # Ok(())
+/// # }
+/// ```
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, TypedBuilder)]
+#[serde(default)]
+#[builder(field_defaults(default, setter(prefix = "with_")))]
+pub struct ArrowReaderOptions {
+ /// Desired batch size, or the reader default when unset.
+ batch_size: Option<usize>,
+ /// Maximum in-flight data files, or execution-host parallelism when unset.
+ data_file_concurrency_limit: Option<usize>,
+ /// Whether to prune row groups using their statistics.
+ #[builder(default = true)]
+ row_group_filtering_enabled: bool,
+ /// Whether to use page indexes for row selection.
+ row_selection_enabled: bool,
+ /// Whether to use bloom filters for row-group pruning.
+ bloom_filter_enabled: bool,
+ /// I/O tuning applied when reading Parquet files.
+ parquet_options: ParquetReadOptions,
+}
+
+impl Default for ArrowReaderOptions {
+ fn default() -> Self {
+ Self::builder().build()
+ }
+}
+
+impl ArrowReaderOptions {
+ /// Returns the configured batch size.
+ pub fn batch_size(&self) -> Option<usize> {
+ self.batch_size
+ }
+
+ pub(super) fn parquet_options_mut(&mut self) -> &mut ParquetReadOptions {
+ &mut self.parquet_options
+ }
+
+ pub(super) fn set_batch_size(&mut self, value: Option<usize>) {
+ self.batch_size = value;
+ }
+ /// Returns the configured data file concurrency limit.
+ pub fn data_file_concurrency_limit(&self) -> Option<usize> {
+ self.data_file_concurrency_limit
+ }
+
+ pub(super) fn set_data_file_concurrency_limit(&mut self, value:
Option<usize>) {
+ self.data_file_concurrency_limit = value;
+ }
+ /// Returns whether row-group statistics filtering is enabled.
+ pub fn row_group_filtering_enabled(&self) -> bool {
+ self.row_group_filtering_enabled
+ }
+
+ pub(super) fn set_row_group_filtering_enabled(&mut self, value: bool) {
+ self.row_group_filtering_enabled = value;
+ }
+ /// Returns whether page-index row selection is enabled.
+ pub fn row_selection_enabled(&self) -> bool {
+ self.row_selection_enabled
+ }
+
+ pub(super) fn set_row_selection_enabled(&mut self, value: bool) {
+ self.row_selection_enabled = value;
+ }
+ /// Returns whether bloom-filter pruning is enabled.
+ pub fn bloom_filter_enabled(&self) -> bool {
+ self.bloom_filter_enabled
+ }
+
+ pub(super) fn set_bloom_filter_enabled(&mut self, value: bool) {
+ self.bloom_filter_enabled = value;
+ }
+ /// Returns the configured parquet options.
+ pub fn parquet_options(&self) -> &ParquetReadOptions {
+ &self.parquet_options
+ }
+
+ pub(super) fn set_parquet_options(&mut self, value: ParquetReadOptions) {
+ self.parquet_options = value;
+ }
+}
+
/// Options for tuning Parquet file I/O.
-#[derive(Clone, Copy, Debug, TypedBuilder)]
+#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize, TypedBuilder)]
Review Comment:
- Copy: removed from ParquetReadOptions; retained Clone.
- PartialEq: derived only under #[cfg(test)] for assertions, no longer part
of the public API.
- *Builder re-exports: removed. Callers still configure options through
ArrowReaderOptions::builder() and ParquetReadOptions::builder().
- Setter consistency: batch size and data-file concurrency now use
strip_option, accepting plain integers. Metadata-size hint is now a plain usize
with a default.`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
