This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-25102-5bae63e7125cf0730c1296b81064d22c93746d75 in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit a78e725e8507eb4b2d5aab55deca309fdc4d0a2c Author: Yongting You <[email protected]> AuthorDate: Thu Sep 10 10:02:16 2026 +0000 docs: Point SessionConfig setter/getter docs to the config definitions (#25102) ## Which issue does this PR close? - Closes #25101. ## Rationale for this change A configuration option's main documentation lives on the config field definition in `datafusion_common::config`, since the user guide (https://datafusion.apache.org/user-guide/configs.html) is generated from there. Some `SessionConfig` setters/getters re-explained the option in their own words, which drifts from the definition over time (and in some cases already had). Keeping the wrapper docs as a plain link to the definition keeps the documentation centralized. ## What changes are included in this PR? Every `SessionConfig` setter/getter that wraps a single config field now uses the same pattern already used by `with_target_partitions` / `target_partitions`: ```rust /// Set [`repartition_joins`] /// /// [`repartition_joins`]: datafusion_common::config::OptimizerOptions::repartition_joins ``` ## What is the testing strategy for this PR? NA ## Are there any user-facing changes? NO --- datafusion/execution/src/config.rs | 180 +++++++++++++++++++++++++------------ 1 file changed, 125 insertions(+), 55 deletions(-) diff --git a/datafusion/execution/src/config.rs b/datafusion/execution/src/config.rs index efaedebdad..80658477c9 100644 --- a/datafusion/execution/src/config.rs +++ b/datafusion/execution/src/config.rs @@ -187,14 +187,22 @@ impl SessionConfig { self } - /// Customize batch size + /// Set [`batch_size`] + /// + /// # Panics + /// + /// Panics if `n` is zero + /// + /// [`batch_size`]: datafusion_common::config::ExecutionOptions::batch_size pub fn with_batch_size(mut self, n: usize) -> Self { self.options_mut().execution.batch_size = ConfigNonZeroUsize::try_new(n).expect("batch size must be greater than zero"); self } - /// Customize [`target_partitions`] + /// Set [`target_partitions`] + /// + /// If `n` is zero, the default value is used instead /// /// [`target_partitions`]: datafusion_common::config::ExecutionOptions::target_partitions pub fn with_target_partitions(mut self, n: usize) -> Self { @@ -219,56 +227,73 @@ impl SessionConfig { self.options.execution.target_partitions } - /// Is the information schema enabled? + /// Get [`information_schema`] + /// + /// [`information_schema`]: datafusion_common::config::CatalogOptions::information_schema pub fn information_schema(&self) -> bool { self.options.catalog.information_schema } - /// Should the context create the default catalog and schema? + /// Get [`create_default_catalog_and_schema`] + /// + /// [`create_default_catalog_and_schema`]: datafusion_common::config::CatalogOptions::create_default_catalog_and_schema pub fn create_default_catalog_and_schema(&self) -> bool { self.options.catalog.create_default_catalog_and_schema } - /// Are joins repartitioned during execution? + /// Get [`repartition_joins`] + /// + /// [`repartition_joins`]: datafusion_common::config::OptimizerOptions::repartition_joins pub fn repartition_joins(&self) -> bool { self.options.optimizer.repartition_joins } - /// Are aggregates repartitioned during execution? + /// Get [`repartition_aggregations`] + /// + /// [`repartition_aggregations`]: datafusion_common::config::OptimizerOptions::repartition_aggregations pub fn repartition_aggregations(&self) -> bool { self.options.optimizer.repartition_aggregations } - /// Are window functions repartitioned during execution? + /// Get [`repartition_windows`] + /// + /// [`repartition_windows`]: datafusion_common::config::OptimizerOptions::repartition_windows pub fn repartition_window_functions(&self) -> bool { self.options.optimizer.repartition_windows } - /// Do we execute sorts in a per-partition fashion and merge afterwards, - /// or do we coalesce partitions first and sort globally? + /// Get [`repartition_sorts`] + /// + /// [`repartition_sorts`]: datafusion_common::config::OptimizerOptions::repartition_sorts pub fn repartition_sorts(&self) -> bool { self.options.optimizer.repartition_sorts } - /// Prefer existing sort (true) or maximize parallelism (false). See - /// [prefer_existing_sort] for more details + /// Get [`prefer_existing_sort`] /// - /// [prefer_existing_sort]: datafusion_common::config::OptimizerOptions::prefer_existing_sort + /// [`prefer_existing_sort`]: datafusion_common::config::OptimizerOptions::prefer_existing_sort pub fn prefer_existing_sort(&self) -> bool { self.options.optimizer.prefer_existing_sort } - /// Are statistics collected during execution? + /// Get [`collect_statistics`] + /// + /// [`collect_statistics`]: datafusion_common::config::ExecutionOptions::collect_statistics pub fn collect_statistics(&self) -> bool { self.options.execution.collect_statistics } - /// Compression codec for spill file + /// Get [`spill_compression`] + /// + /// [`spill_compression`]: datafusion_common::config::ExecutionOptions::spill_compression pub fn spill_compression(&self) -> SpillCompression { self.options.execution.spill_compression } - /// Selects a name for the default catalog and schema + /// Set [`default_catalog`] and [`default_schema`] + /// + /// [`default_catalog`]: datafusion_common::config::CatalogOptions::default_catalog + /// [`default_schema`]: datafusion_common::config::CatalogOptions::default_schema pub fn with_default_catalog_and_schema( mut self, catalog: impl Into<String>, @@ -279,37 +304,49 @@ impl SessionConfig { self } - /// Controls whether the default catalog and schema will be automatically created + /// Set [`create_default_catalog_and_schema`] + /// + /// [`create_default_catalog_and_schema`]: datafusion_common::config::CatalogOptions::create_default_catalog_and_schema pub fn with_create_default_catalog_and_schema(mut self, create: bool) -> Self { self.options_mut().catalog.create_default_catalog_and_schema = create; self } - /// Enables or disables the inclusion of `information_schema` virtual tables + /// Set [`information_schema`] + /// + /// [`information_schema`]: datafusion_common::config::CatalogOptions::information_schema pub fn with_information_schema(mut self, enabled: bool) -> Self { self.options_mut().catalog.information_schema = enabled; self } - /// Enables or disables the use of repartitioning for joins to improve parallelism + /// Set [`repartition_joins`] + /// + /// [`repartition_joins`]: datafusion_common::config::OptimizerOptions::repartition_joins pub fn with_repartition_joins(mut self, enabled: bool) -> Self { self.options_mut().optimizer.repartition_joins = enabled; self } - /// Enables or disables the use of repartitioning for aggregations to improve parallelism + /// Set [`repartition_aggregations`] + /// + /// [`repartition_aggregations`]: datafusion_common::config::OptimizerOptions::repartition_aggregations pub fn with_repartition_aggregations(mut self, enabled: bool) -> Self { self.options_mut().optimizer.repartition_aggregations = enabled; self } - /// Sets minimum file range size for repartitioning scans + /// Set [`repartition_file_min_size`] + /// + /// [`repartition_file_min_size`]: datafusion_common::config::OptimizerOptions::repartition_file_min_size pub fn with_repartition_file_min_size(mut self, size: usize) -> Self { self.options_mut().optimizer.repartition_file_min_size = size; self } - /// Enables or disables the allowing unordered symmetric hash join + /// Set [`allow_symmetric_joins_without_pruning`] + /// + /// [`allow_symmetric_joins_without_pruning`]: datafusion_common::config::OptimizerOptions::allow_symmetric_joins_without_pruning pub fn with_allow_symmetric_joins_without_pruning(mut self, enabled: bool) -> Self { self.options_mut() .optimizer @@ -317,118 +354,147 @@ impl SessionConfig { self } - /// Enables or disables the use of repartitioning for file scans + /// Set [`repartition_file_scans`] + /// + /// [`repartition_file_scans`]: datafusion_common::config::OptimizerOptions::repartition_file_scans pub fn with_repartition_file_scans(mut self, enabled: bool) -> Self { self.options_mut().optimizer.repartition_file_scans = enabled; self } - /// Enables or disables the use of repartitioning for window functions to improve parallelism + /// Set [`repartition_windows`] + /// + /// [`repartition_windows`]: datafusion_common::config::OptimizerOptions::repartition_windows pub fn with_repartition_windows(mut self, enabled: bool) -> Self { self.options_mut().optimizer.repartition_windows = enabled; self } - /// Enables or disables the use of per-partition sorting to improve parallelism + /// Set [`repartition_sorts`] + /// + /// [`repartition_sorts`]: datafusion_common::config::OptimizerOptions::repartition_sorts pub fn with_repartition_sorts(mut self, enabled: bool) -> Self { self.options_mut().optimizer.repartition_sorts = enabled; self } - /// Prefer existing sort (true) or maximize parallelism (false). See - /// [prefer_existing_sort] for more details + /// Set [`prefer_existing_sort`] /// - /// [prefer_existing_sort]: datafusion_common::config::OptimizerOptions::prefer_existing_sort + /// [`prefer_existing_sort`]: datafusion_common::config::OptimizerOptions::prefer_existing_sort pub fn with_prefer_existing_sort(mut self, enabled: bool) -> Self { self.options_mut().optimizer.prefer_existing_sort = enabled; self } - /// Prefer existing union (true). See [prefer_existing_union] for more details + /// Set [`prefer_existing_union`] /// - /// [prefer_existing_union]: datafusion_common::config::OptimizerOptions::prefer_existing_union + /// [`prefer_existing_union`]: datafusion_common::config::OptimizerOptions::prefer_existing_union pub fn with_prefer_existing_union(mut self, enabled: bool) -> Self { self.options_mut().optimizer.prefer_existing_union = enabled; self } - /// Enables or disables the use of pruning predicate for parquet readers to skip row groups + /// Set [`pruning`] + /// + /// [`pruning`]: datafusion_common::config::ParquetOptions::pruning pub fn with_parquet_pruning(mut self, enabled: bool) -> Self { self.options_mut().execution.parquet.pruning = enabled; self } - /// Returns true if pruning predicate should be used to skip parquet row groups + /// Get [`pruning`] + /// + /// [`pruning`]: datafusion_common::config::ParquetOptions::pruning pub fn parquet_pruning(&self) -> bool { self.options.execution.parquet.pruning } - /// Returns true if bloom filter should be used to skip parquet row groups + /// Get [`bloom_filter_on_read`] + /// + /// [`bloom_filter_on_read`]: datafusion_common::config::ParquetOptions::bloom_filter_on_read pub fn parquet_bloom_filter_pruning(&self) -> bool { self.options.execution.parquet.bloom_filter_on_read } - /// Enables or disables the use of bloom filter for parquet readers to skip row groups + /// Set [`bloom_filter_on_read`] + /// + /// [`bloom_filter_on_read`]: datafusion_common::config::ParquetOptions::bloom_filter_on_read pub fn with_parquet_bloom_filter_pruning(mut self, enabled: bool) -> Self { self.options_mut().execution.parquet.bloom_filter_on_read = enabled; self } - /// Returns true if page index should be used to skip parquet data pages + /// Get [`enable_page_index`] + /// + /// [`enable_page_index`]: datafusion_common::config::ParquetOptions::enable_page_index pub fn parquet_page_index_pruning(&self) -> bool { self.options.execution.parquet.enable_page_index } - /// Enables or disables the use of page index for parquet readers to skip parquet data pages + /// Set [`enable_page_index`] + /// + /// [`enable_page_index`]: datafusion_common::config::ParquetOptions::enable_page_index pub fn with_parquet_page_index_pruning(mut self, enabled: bool) -> Self { self.options_mut().execution.parquet.enable_page_index = enabled; self } - /// Enables or disables the collection of statistics after listing files + /// Set [`collect_statistics`] + /// + /// [`collect_statistics`]: datafusion_common::config::ExecutionOptions::collect_statistics pub fn with_collect_statistics(mut self, enabled: bool) -> Self { self.options_mut().execution.collect_statistics = enabled; self } - /// Get the currently configured batch size + /// Get [`batch_size`] + /// + /// [`batch_size`]: datafusion_common::config::ExecutionOptions::batch_size pub fn batch_size(&self) -> usize { self.options.execution.batch_size.get() } - /// Enables or disables the coalescence of small batches into larger batches + /// Set [`coalesce_batches`] + /// + /// [`coalesce_batches`]: datafusion_common::config::ExecutionOptions::coalesce_batches pub fn with_coalesce_batches(mut self, enabled: bool) -> Self { self.options_mut().execution.coalesce_batches = enabled; self } - /// Returns true if record batches will be examined between each operator - /// and small batches will be coalesced into larger batches. + /// Get [`coalesce_batches`] + /// + /// [`coalesce_batches`]: datafusion_common::config::ExecutionOptions::coalesce_batches pub fn coalesce_batches(&self) -> bool { self.options.execution.coalesce_batches } - /// Enables or disables the round robin repartition for increasing parallelism + /// Set [`enable_round_robin_repartition`] + /// + /// [`enable_round_robin_repartition`]: datafusion_common::config::OptimizerOptions::enable_round_robin_repartition pub fn with_round_robin_repartition(mut self, enabled: bool) -> Self { self.options_mut().optimizer.enable_round_robin_repartition = enabled; self } - /// Returns true if the physical plan optimizer will try to - /// add round robin repartition to increase parallelism to leverage more CPU cores. + /// Get [`enable_round_robin_repartition`] + /// + /// [`enable_round_robin_repartition`]: datafusion_common::config::OptimizerOptions::enable_round_robin_repartition pub fn round_robin_repartition(&self) -> bool { self.options.optimizer.enable_round_robin_repartition } - /// Enables or disables sort pushdown optimization, and currently only - /// applies to Parquet data source. + /// Set [`enable_sort_pushdown`] + /// + /// [`enable_sort_pushdown`]: datafusion_common::config::OptimizerOptions::enable_sort_pushdown pub fn with_enable_sort_pushdown(mut self, enabled: bool) -> Self { self.options_mut().optimizer.enable_sort_pushdown = enabled; self } - /// Enables or disables elimination of `ORDER BY` clauses in subqueries - /// when they are not required by order-sensitive operators. + /// Set [`enable_subquery_sort_elimination`] + /// + /// [`enable_subquery_sort_elimination`]: datafusion_common::config::SqlParserOptions::enable_subquery_sort_elimination pub fn with_enable_subquery_sort_elimination(mut self, enabled: bool) -> Self { self.options_mut() .sql_parser @@ -436,8 +502,7 @@ impl SessionConfig { self } - /// Set the size of [`sort_spill_reservation_bytes`] to control - /// memory pre-reservation + /// Set [`sort_spill_reservation_bytes`] /// /// [`sort_spill_reservation_bytes`]: datafusion_common::config::ExecutionOptions::sort_spill_reservation_bytes pub fn with_sort_spill_reservation_bytes( @@ -449,7 +514,7 @@ impl SessionConfig { self } - /// Set the compression codec [`spill_compression`] used when spilling data to disk. + /// Set [`spill_compression`] /// /// [`spill_compression`]: datafusion_common::config::ExecutionOptions::spill_compression pub fn with_spill_compression(mut self, spill_compression: SpillCompression) -> Self { @@ -457,8 +522,7 @@ impl SessionConfig { self } - /// Set the size of [`sort_in_place_threshold_bytes`] to control - /// how sort does things. + /// Set [`sort_in_place_threshold_bytes`] /// /// [`sort_in_place_threshold_bytes`]: datafusion_common::config::ExecutionOptions::sort_in_place_threshold_bytes pub fn with_sort_in_place_threshold_bytes( @@ -470,7 +534,9 @@ impl SessionConfig { self } - /// Enables or disables the enforcement of batch size in joins + /// Set [`enforce_batch_size_in_joins`] + /// + /// [`enforce_batch_size_in_joins`]: datafusion_common::config::ExecutionOptions::enforce_batch_size_in_joins pub fn with_enforce_batch_size_in_joins( mut self, enforce_batch_size_in_joins: bool, @@ -480,12 +546,16 @@ impl SessionConfig { self } - /// Returns true if the joins will be enforced to output batches of the configured size + /// Get [`enforce_batch_size_in_joins`] + /// + /// [`enforce_batch_size_in_joins`]: datafusion_common::config::ExecutionOptions::enforce_batch_size_in_joins pub fn enforce_batch_size_in_joins(&self) -> bool { self.options.execution.enforce_batch_size_in_joins } - /// Toggle SQL ANSI mode for expressions, casting, and error handling + /// Set [`enable_ansi_mode`] + /// + /// [`enable_ansi_mode`]: datafusion_common::config::ExecutionOptions::enable_ansi_mode pub fn with_enable_ansi_mode(mut self, enable_ansi_mode: bool) -> Self { self.options_mut().execution.enable_ansi_mode = enable_ansi_mode; self --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
