This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/asf-site by this push:
new c8fa1197cf Publish built docs triggered by
957886db83c631027fffd11224a50de233f7bcf6
c8fa1197cf is described below
commit c8fa1197cf0b2e715e78a2fa0e103fbf609f593b
Author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Jul 20 21:49:28 2026 +0000
Publish built docs triggered by 957886db83c631027fffd11224a50de233f7bcf6
---
.../benchmark-results/tpc-h.md.txt | 2 +-
.../contributor-guide/benchmarking_macos.md.txt | 2 +-
.../contributor-guide/config_conventions.md.txt | 115 +++++++++++++
_sources/contributor-guide/index.md.txt | 1 +
_sources/user-guide/latest/configs.md.txt | 2 +-
_sources/user-guide/latest/tuning.md.txt | 2 +-
contributor-guide/adding_a_new_expression.html | 1 +
contributor-guide/adding_a_new_operator.html | 1 +
contributor-guide/adding_a_new_spark_version.html | 1 +
contributor-guide/benchmark-results/tpc-ds.html | 1 +
contributor-guide/benchmark-results/tpc-h.html | 3 +-
contributor-guide/benchmarking.html | 1 +
contributor-guide/benchmarking_aws_ec2.html | 1 +
contributor-guide/benchmarking_macos.html | 3 +-
contributor-guide/benchmarking_spark_sql_perf.html | 1 +
contributor-guide/bug_triage.html | 7 +-
...rg-spark-tests.html => config_conventions.html} | 188 +++++++++++----------
contributor-guide/contributing.html | 1 +
contributor-guide/debugging.html | 1 +
contributor-guide/development.html | 1 +
contributor-guide/expression-audits/agg_funcs.html | 1 +
.../expression-audits/array_funcs.html | 1 +
.../expression-audits/bitwise_funcs.html | 1 +
.../expression-audits/collection_funcs.html | 1 +
.../expression-audits/conditional_funcs.html | 1 +
.../expression-audits/conversion_funcs.html | 1 +
.../expression-audits/datetime_funcs.html | 1 +
.../expression-audits/generator_funcs.html | 1 +
.../expression-audits/hash_funcs.html | 1 +
contributor-guide/expression-audits/index.html | 1 +
.../expression-audits/json_funcs.html | 1 +
contributor-guide/expression-audits/map_funcs.html | 1 +
.../expression-audits/math_funcs.html | 1 +
.../expression-audits/misc_funcs.html | 1 +
.../expression-audits/predicate_funcs.html | 1 +
.../expression-audits/string_funcs.html | 1 +
.../expression-audits/struct_funcs.html | 1 +
contributor-guide/expression-audits/url_funcs.html | 1 +
.../expression-audits/window_funcs.html | 1 +
contributor-guide/ffi.html | 1 +
contributor-guide/iceberg-spark-tests.html | 1 +
contributor-guide/index.html | 1 +
contributor-guide/jvm_shuffle.html | 1 +
contributor-guide/native_shuffle.html | 1 +
contributor-guide/optimizing_expressions.html | 1 +
contributor-guide/plugin_overview.html | 1 +
contributor-guide/profiling.html | 1 +
contributor-guide/release_process.html | 1 +
contributor-guide/roadmap.html | 1 +
.../s3-credential-provider-design.html | 1 +
contributor-guide/spark-sql-tests.html | 1 +
contributor-guide/spark_configs_support.html | 7 +-
contributor-guide/sql-file-tests.html | 1 +
contributor-guide/sql_error_propagation.html | 1 +
contributor-guide/tracing.html | 1 +
genindex.html | 1 +
objects.inv | Bin 3785 -> 3811 bytes
search.html | 1 +
searchindex.js | 2 +-
.../expressions/_category_template/aggregate.html | 1 +
.../expressions/_category_template/array.html | 1 +
.../expressions/_category_template/cast.html | 1 +
.../expressions/_category_template/datetime.html | 1 +
.../expressions/_category_template/map.html | 1 +
.../expressions/_category_template/math.html | 1 +
.../expressions/_category_template/misc.html | 1 +
.../expressions/_category_template/string.html | 1 +
.../expressions/_category_template/struct.html | 1 +
.../expressions/_category_template/url.html | 1 +
user-guide/latest/configs.html | 8 +-
user-guide/latest/tuning.html | 2 +-
71 files changed, 296 insertions(+), 104 deletions(-)
diff --git a/_sources/contributor-guide/benchmark-results/tpc-h.md.txt
b/_sources/contributor-guide/benchmark-results/tpc-h.md.txt
index c489f8c999..eab15fb84a 100644
--- a/_sources/contributor-guide/benchmark-results/tpc-h.md.txt
+++ b/_sources/contributor-guide/benchmark-results/tpc-h.md.txt
@@ -74,6 +74,6 @@ spark.memory.offHeap.size=32G
### Comet (Tuned)
```properties
-spark.comet.exec.replaceSortMergeJoin=true
+spark.comet.exec.forceShuffledHashJoin=true
spark.comet.memoryPool.fraction=0.8
```
diff --git a/_sources/contributor-guide/benchmarking_macos.md.txt
b/_sources/contributor-guide/benchmarking_macos.md.txt
index 5e98acaae9..70135ed210 100644
--- a/_sources/contributor-guide/benchmarking_macos.md.txt
+++ b/_sources/contributor-guide/benchmarking_macos.md.txt
@@ -154,7 +154,7 @@ $SPARK_HOME/bin/spark-submit \
--conf spark.comet.enabled=true \
--conf spark.comet.exec.shuffle.enableFastEncoding=true \
--conf spark.comet.exec.shuffle.fallbackToColumnar=true \
- --conf spark.comet.exec.replaceSortMergeJoin=true \
+ --conf spark.comet.exec.forceShuffledHashJoin=true \
--conf spark.comet.expression.allowIncompatible=true \
$DF_BENCH/runners/datafusion-comet/tpcbench.py \
--benchmark tpch \
diff --git a/_sources/contributor-guide/config_conventions.md.txt
b/_sources/contributor-guide/config_conventions.md.txt
new file mode 100644
index 0000000000..0e01c958c1
--- /dev/null
+++ b/_sources/contributor-guide/config_conventions.md.txt
@@ -0,0 +1,115 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Configuration Conventions
+
+Comet configuration keys live under `spark.comet.*` and are declared in
+`spark/src/main/scala/org/apache/comet/CometConf.scala`. This page describes
the naming
+conventions those keys follow so that new configs stay consistent with the
surface users
+already know, and documents the process for renaming an existing key without
breaking
+deployments.
+
+## Key Naming
+
+A Comet config key is a dotted path of segments. Each segment describes a
scope, from
+broadest (the prefix) to narrowest (the specific setting).
+
+- **Prefix.** All keys start with `spark.comet.` — never bare `comet.` and
never a nested
+ prefix like `spark.sql.comet.`.
+- **Category segment.** The first segment after the prefix names a broad area:
+ `exec` (execution and expressions), `scan` (source readers), `parquet`
(Parquet-specific
+ settings), `shuffle` (shuffle behavior), `columnar` (columnar shuffle),
`explain` (plan
+ explain/logging), `metrics`, `tracing`, `testing`, `convert` (Spark → Comet
source
+ conversion), or `expression` / `operator` (per-expression / per-operator
flags).
+- **Segment casing.** Multi-word segments use `camelCase`, not dot- or
kebab-separated.
+ Prefer `spark.comet.foo.maxThreadNum` over `spark.comet.foo.max.thread.num`
or
+ `spark.comet.foo.max-thread-num`.
+- **Boolean suffix.** Descriptor-form boolean flags that gate a subsystem or
feature end
+ in `.enabled` — for example `spark.comet.debug.enabled`,
`spark.comet.metrics.enabled`,
+ `spark.comet.parquet.rowFilterPushdown.enabled`. Action-form flags whose
name is itself
+ a verb (`force...`, `allow...`) can omit `.enabled` because the verb already
encodes
+ the action — for example `spark.comet.exec.forceShuffledHashJoin`.
+- **Acronyms.** Treat acronyms consistently as `UDF`, `SHJ`, `SMJ`, `IO` —
either all-caps
+ or camelCase, but do not mix within one key (`pyarrowUdf` and `scalaUDF` in
the same
+ cluster is a red flag).
+
+## Symbol Naming (Scala)
+
+Every config declares a Scala `val` in `CometConf.scala`. The symbol name is
the
+`UPPER_SNAKE_CASE` form of the key's meaningful segments, prefixed with
`COMET_`.
+
+Examples:
+
+| Key | Symbol
|
+| ----------------------------------------------- |
--------------------------- |
+| `spark.comet.enabled` | `COMET_ENABLED`
|
+| `spark.comet.exec.forceShuffledHashJoin` | `COMET_FORCE_SHJ`
|
+| `spark.comet.parquet.rowFilterPushdown.enabled` |
`COMET_ROW_FILTER_PUSHDOWN` |
+
+The symbol name is what appears in code; the key is what appears in user
configuration.
+The two do not have to match segment-for-segment — brevity in the symbol is
fine as long
+as the key remains descriptive.
+
+## Categories
+
+Every `ConfigEntry` must call `.category(...)`. The category is used to route
the key into
+the right table in the user guide's `configs.md`. Available categories are
declared as
+`CATEGORY_*` constants at the top of `CometConf.scala`. If a new config does
not fit an
+existing category, discuss adding a new one before landing the config.
+
+## Renaming an Existing Config
+
+Configs under `spark.comet.*` are stable across minor releases: users may have
set them in
+production `spark-defaults.conf` files, Spark job submissions, or notebooks.
Renaming a key
+must not silently break those deployments.
+
+Use the `withAlternative` builder on `ConfigBuilder` to keep the old key
working as a
+deprecated alias:
+
+```scala
+val COMET_FORCE_SHJ: ConfigEntry[Boolean] =
+ conf(s"$COMET_EXEC_CONFIG_PREFIX.forceShuffledHashJoin")
+ .withAlternative(s"$COMET_EXEC_CONFIG_PREFIX.replaceSortMergeJoin")
+ .category(CATEGORY_EXEC)
+ .doc("...")
+ .booleanConf
+ .createWithDefault(false)
+```
+
+Reading a value from the alternative logs a one-time deprecation warning per
JVM per
+alternative key, pointing users at the current key. The primary key always
wins if both
+are set.
+
+`withAlternative` accepts multiple alternatives (checked in order), for keys
that have
+been renamed more than once.
+
+The rename checklist for a single config:
+
+1. Update the key string on the `conf(...)` call and add
`.withAlternative(oldKey)`.
+2. Rename the Scala `val` (for example `COMET_REPLACE_SMJ` →
`COMET_FORCE_SHJ`).
+3. Update every callsite of the Scala symbol — grep the whole repo.
+4. Update every documented reference to the old key string — grep
`docs/source/` for the
+ old key and update to the new key. The auto-generated `configs.md` table
will refresh
+ on the next release-docs regeneration and does not need manual editing.
+5. Add or update a test in `CometConfSuite` if the rename covers a new type of
alias
+ pattern (single-hop, multiple alternatives, etc.).
+
+Removing a deprecated alias is a follow-up step that belongs to a later major
release —
+typically the next Comet major after the rename first ships. See the
+[versioning policy](../about/versioning_policy.md) for the timing rules.
diff --git a/_sources/contributor-guide/index.md.txt
b/_sources/contributor-guide/index.md.txt
index f293961bda..397239dbe4 100644
--- a/_sources/contributor-guide/index.md.txt
+++ b/_sources/contributor-guide/index.md.txt
@@ -91,6 +91,7 @@ Tracing <tracing>
Expression Audits <expression-audits/index>
Supported Spark Configurations <spark_configs_support>
+Configuration Conventions <config_conventions>
```
```{toctree}
diff --git a/_sources/user-guide/latest/configs.md.txt
b/_sources/user-guide/latest/configs.md.txt
index c91e4bfe22..f79e45df39 100644
--- a/_sources/user-guide/latest/configs.md.txt
+++ b/_sources/user-guide/latest/configs.md.txt
@@ -60,8 +60,8 @@ Comet provides the following configuration settings.
| `spark.comet.exceptionOnDatetimeRebase` | Whether to throw exception when
seeing dates/timestamps from the legacy hybrid (Julian + Gregorian) calendar.
Since Spark 3, dates/timestamps were written according to the Proleptic
Gregorian calendar. When this is true, Comet will throw exceptions when seeing
these dates/timestamps that were written by Spark version before 3.0. If this
is false, these dates/timestamps will be read as if they were written to the
Proleptic Gregorian calendar and [...]
| `spark.comet.exec.columnarToRow.native.enabled` | Whether to enable native
columnar to row conversion. When enabled, Comet will use native Rust code to
convert Arrow columnar data to Spark UnsafeRow format instead of the JVM
implementation. This can improve performance for queries that need to convert
between columnar and row formats. | true |
| `spark.comet.exec.enabled` | Whether to enable Comet native vectorized
execution for Spark. This controls whether Spark should convert operators into
their Comet counterparts and execute them in native space. Note: each operator
is associated with a separate config in the format of
`spark.comet.exec.<operator_name>.enabled` at the moment, and both the config
and this need to be turned on, in order for the operator to be executed in
native. | true |
+| `spark.comet.exec.forceShuffledHashJoin` | Experimental feature to force
Spark to replace SortMergeJoin with ShuffledHashJoin for improved performance.
This feature is not stable yet. For more information, refer to the [Comet
Tuning
Guide](https://datafusion.apache.org/comet/user-guide/latest/tuning.html). |
false |
| `spark.comet.exec.pyarrowUdf.enabled` | Experimental: whether to enable
optimized execution of PyArrow UDFs (mapInArrow/mapInPandas). When enabled,
Comet passes Arrow columnar data directly to Python UDFs without the
intermediate Arrow-to-Row-to-Arrow conversion that Spark normally performs.
Disabled by default while the feature stabilizes. | false |
-| `spark.comet.exec.replaceSortMergeJoin` | Experimental feature to force
Spark to replace SortMergeJoin with ShuffledHashJoin for improved performance.
This feature is not stable yet. For more information, refer to the [Comet
Tuning
Guide](https://datafusion.apache.org/comet/user-guide/latest/tuning.html). |
false |
| `spark.comet.exec.scalaUDF.codegen.enabled` | Whether to route Spark
`ScalaUDF` expressions through Comet's Arrow-direct codegen dispatcher. When
enabled, a supported ScalaUDF is compiled into a per-batch kernel that reads
and writes Arrow vectors directly from native execution. When disabled, plans
containing a ScalaUDF fall back to Spark for the enclosing operator. The same
dispatcher backs the regex family (`rlike`, `regexp_replace`, `split`,
`regexp_extract`, `regexp_extract_all`, [...]
| `spark.comet.exec.strictFloatingPoint` | When enabled, fall back to Spark
for floating-point operations that may differ from Spark, such as when
comparing or sorting -0.0 and 0.0. For more information, refer to the [Comet
Compatibility
Guide](https://datafusion.apache.org/comet/user-guide/latest/compatibility/index.html).
| false |
| `spark.comet.exec.transitionRevert.enabled` | When enabled, Comet reverts a
query stage to Spark row-based execution if the number of columnar-to-row (C2R)
transitions in the stage exceeds the configured threshold. This avoids the
overhead of repeated format conversions in stages where many operators fall
back to row-based execution. | false |
diff --git a/_sources/user-guide/latest/tuning.md.txt
b/_sources/user-guide/latest/tuning.md.txt
index 5fa9c5f344..6f0e9a684b 100644
--- a/_sources/user-guide/latest/tuning.md.txt
+++ b/_sources/user-guide/latest/tuning.md.txt
@@ -119,7 +119,7 @@ to configure Comet to convert `SortMergeJoin` to
`ShuffledHashJoin`. Comet does
`ShuffledHashJoin` so this could result in OOM. Also, `SortMergeJoin` may
still be faster in some cases. It is best
to test with both for your specific workloads.
-To configure Comet to convert `SortMergeJoin` to `ShuffledHashJoin`, set
`spark.comet.exec.replaceSortMergeJoin=true`.
+To configure Comet to convert `SortMergeJoin` to `ShuffledHashJoin`, set
`spark.comet.exec.forceShuffledHashJoin=true`.
## Shuffle
diff --git a/contributor-guide/adding_a_new_expression.html
b/contributor-guide/adding_a_new_expression.html
index dc6214f136..6f69a31bd2 100644
--- a/contributor-guide/adding_a_new_expression.html
+++ b/contributor-guide/adding_a_new_expression.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/adding_a_new_operator.html
b/contributor-guide/adding_a_new_operator.html
index e95962c844..7698f99a08 100644
--- a/contributor-guide/adding_a_new_operator.html
+++ b/contributor-guide/adding_a_new_operator.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/adding_a_new_spark_version.html
b/contributor-guide/adding_a_new_spark_version.html
index 66eb856a71..0ce8329638 100644
--- a/contributor-guide/adding_a_new_spark_version.html
+++ b/contributor-guide/adding_a_new_spark_version.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/benchmark-results/tpc-ds.html
b/contributor-guide/benchmark-results/tpc-ds.html
index 2788e94d39..aec0822a1c 100644
--- a/contributor-guide/benchmark-results/tpc-ds.html
+++ b/contributor-guide/benchmark-results/tpc-ds.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/benchmark-results/tpc-h.html
b/contributor-guide/benchmark-results/tpc-h.html
index a1125e8733..206bccb50a 100644
--- a/contributor-guide/benchmark-results/tpc-h.html
+++ b/contributor-guide/benchmark-results/tpc-h.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
@@ -587,7 +588,7 @@ under the License.
</div>
<section id="comet-tuned">
<h3>Comet (Tuned)<a class="headerlink" href="#comet-tuned" title="Link to this
heading">#</a></h3>
-<div class="highlight-properties notranslate"><div
class="highlight"><pre><span></span><span
class="na">spark.comet.exec.replaceSortMergeJoin</span><span
class="o">=</span><span class="s">true</span>
+<div class="highlight-properties notranslate"><div
class="highlight"><pre><span></span><span
class="na">spark.comet.exec.forceShuffledHashJoin</span><span
class="o">=</span><span class="s">true</span>
<span class="na">spark.comet.memoryPool.fraction</span><span
class="o">=</span><span class="s">0.8</span>
</pre></div>
</div>
diff --git a/contributor-guide/benchmarking.html
b/contributor-guide/benchmarking.html
index 66bc7215e5..46cdeb00be 100644
--- a/contributor-guide/benchmarking.html
+++ b/contributor-guide/benchmarking.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/benchmarking_aws_ec2.html
b/contributor-guide/benchmarking_aws_ec2.html
index b55df98205..14c2279f2e 100644
--- a/contributor-guide/benchmarking_aws_ec2.html
+++ b/contributor-guide/benchmarking_aws_ec2.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/benchmarking_macos.html
b/contributor-guide/benchmarking_macos.html
index 80747d6c5b..2786a95dcf 100644
--- a/contributor-guide/benchmarking_macos.html
+++ b/contributor-guide/benchmarking_macos.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
@@ -663,7 +664,7 @@ In order to start local Apache Spark cluster on <code
class="docutils literal no
<span class="w"> </span>--conf<span class="w">
</span>spark.comet.enabled<span class="o">=</span><span
class="nb">true</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w">
</span>spark.comet.exec.shuffle.enableFastEncoding<span class="o">=</span><span
class="nb">true</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w">
</span>spark.comet.exec.shuffle.fallbackToColumnar<span class="o">=</span><span
class="nb">true</span><span class="w"> </span><span class="se">\</span>
-<span class="w"> </span>--conf<span class="w">
</span>spark.comet.exec.replaceSortMergeJoin<span class="o">=</span><span
class="nb">true</span><span class="w"> </span><span class="se">\</span>
+<span class="w"> </span>--conf<span class="w">
</span>spark.comet.exec.forceShuffledHashJoin<span class="o">=</span><span
class="nb">true</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>--conf<span class="w">
</span>spark.comet.expression.allowIncompatible<span class="o">=</span><span
class="nb">true</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span><span
class="nv">$DF_BENCH</span>/runners/datafusion-comet/tpcbench.py<span
class="w"> </span><span class="se">\</span>
<span class="w"> </span>--benchmark<span class="w"> </span>tpch<span
class="w"> </span><span class="se">\</span>
diff --git a/contributor-guide/benchmarking_spark_sql_perf.html
b/contributor-guide/benchmarking_spark_sql_perf.html
index 8232df1e27..8dd13cd3cc 100644
--- a/contributor-guide/benchmarking_spark_sql_perf.html
+++ b/contributor-guide/benchmarking_spark_sql_perf.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/bug_triage.html
b/contributor-guide/bug_triage.html
index 2a3a26cf9e..024abc9f8a 100644
--- a/contributor-guide/bug_triage.html
+++ b/contributor-guide/bug_triage.html
@@ -69,7 +69,7 @@ under the License.
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="Apache DataFusion Comet: Release Process"
href="release_process.html" />
- <link rel="prev" title="Supported Spark Configurations"
href="spark_configs_support.html" />
+ <link rel="prev" title="Configuration Conventions"
href="config_conventions.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="current nav bd-sidenav">
@@ -782,12 +783,12 @@ should be addressed before the writer is promoted to
production-ready status.</p
<div class="prev-next-area">
<a class="left-prev"
- href="spark_configs_support.html"
+ href="config_conventions.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Supported Spark Configurations</p>
+ <p class="prev-next-title">Configuration Conventions</p>
</div>
</a>
<a class="right-next"
diff --git a/contributor-guide/iceberg-spark-tests.html
b/contributor-guide/config_conventions.html
similarity index 67%
copy from contributor-guide/iceberg-spark-tests.html
copy to contributor-guide/config_conventions.html
index 03d14f67e6..840eb1a551 100644
--- a/contributor-guide/iceberg-spark-tests.html
+++ b/contributor-guide/config_conventions.html
@@ -29,7 +29,7 @@ under the License.
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0"
/><meta name="viewport" content="width=device-width, initial-scale=1" />
- <title>Running Iceberg Spark Tests — Apache DataFusion Comet
documentation</title>
+ <title>Configuration Conventions — Apache DataFusion Comet
documentation</title>
@@ -63,13 +63,13 @@ under the License.
<script src="../_static/documentation_options.js?v=5929fcd5"></script>
<script src="../_static/doctools.js?v=9a2dae69"></script>
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
- <script>DOCUMENTATION_OPTIONS.pagename =
'contributor-guide/iceberg-spark-tests';</script>
+ <script>DOCUMENTATION_OPTIONS.pagename =
'contributor-guide/config_conventions';</script>
<script async="true" defer="true"
src="https://buttons.github.io/buttons.js";></script>
<script src="../_static/comet-ux.js?v=97577a30"></script>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
- <link rel="next" title="Comet Debugging Guide" href="debugging.html" />
- <link rel="prev" title="Running Spark SQL Tests"
href="spark-sql-tests.html" />
+ <link rel="next" title="Bug Triage Guide" href="bug_triage.html" />
+ <link rel="prev" title="Supported Spark Configurations"
href="spark_configs_support.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
@@ -412,10 +412,10 @@ under the License.
<li class="toctree-l1"><a class="reference internal"
href="adding_a_new_spark_version.html">Adding a New Spark Version</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Testing</span></p>
-<ul class="current nav bd-sidenav">
+<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal"
href="sql-file-tests.html">Comet SQL Tests</a></li>
<li class="toctree-l1"><a class="reference internal"
href="spark-sql-tests.html">Spark SQL Tests</a></li>
-<li class="toctree-l1 current active"><a class="current reference internal"
href="#">Iceberg Spark Tests</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="iceberg-spark-tests.html">Iceberg Spark Tests</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Debugging and Performance</span></p>
<ul class="nav bd-sidenav">
@@ -433,7 +433,7 @@ under the License.
<li class="toctree-l1"><a class="reference internal"
href="tracing.html">Tracing</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Reference</span></p>
-<ul class="nav bd-sidenav">
+<ul class="current nav bd-sidenav">
<li class="toctree-l1 has-children"><a class="reference internal"
href="expression-audits/index.html">Expression
Audits</a><details><summary><span class="toctree-toggle" role="presentation"><i
class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal"
href="expression-audits/agg_funcs.html">agg_funcs</a></li>
<li class="toctree-l2"><a class="reference internal"
href="expression-audits/array_funcs.html">array_funcs</a></li>
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1 current active"><a class="current reference internal"
href="#">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
@@ -508,7 +509,7 @@ under the License.
<li class="breadcrumb-item"><a href="index.html" class="nav-link">Comet
Contributor Guide</a></li>
- <li class="breadcrumb-item active" aria-current="page"><span
class="ellipsis">Running Iceberg Spark Tests</span></li>
+ <li class="breadcrumb-item active" aria-current="page"><span
class="ellipsis">Configuration Conventions</span></li>
</ul>
</nav>
</div>
@@ -543,92 +544,108 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<section id="running-iceberg-spark-tests">
-<h1>Running Iceberg Spark Tests<a class="headerlink"
href="#running-iceberg-spark-tests" title="Link to this heading">#</a></h1>
-<p>Running Apache Iceberg’s Spark tests with Comet enabled is a good way to
ensure that Comet produces the same
-results as Spark when reading Iceberg tables. To enable this, we apply diff
files to the Apache Iceberg source
-code so that Comet is loaded when we run the tests.</p>
-<p>Here is an overview of the changes that the diffs make to Iceberg:</p>
+<section id="configuration-conventions">
+<h1>Configuration Conventions<a class="headerlink"
href="#configuration-conventions" title="Link to this heading">#</a></h1>
+<p>Comet configuration keys live under <code class="docutils literal
notranslate"><span class="pre">spark.comet.*</span></code> and are declared in
+<code class="docutils literal notranslate"><span
class="pre">spark/src/main/scala/org/apache/comet/CometConf.scala</span></code>.
This page describes the naming
+conventions those keys follow so that new configs stay consistent with the
surface users
+already know, and documents the process for renaming an existing key without
breaking
+deployments.</p>
+<section id="key-naming">
+<h2>Key Naming<a class="headerlink" href="#key-naming" title="Link to this
heading">#</a></h2>
+<p>A Comet config key is a dotted path of segments. Each segment describes a
scope, from
+broadest (the prefix) to narrowest (the specific setting).</p>
<ul class="simple">
-<li><p>Configure Comet as a dependency and set the correct version in <code
class="docutils literal notranslate"><span
class="pre">libs.versions.toml</span></code> and <code class="docutils literal
notranslate"><span class="pre">build.gradle</span></code></p></li>
-<li><p>Delete upstream Comet reader classes that reference legacy Comet APIs
removed in <a class="reference external"
href="https://github.com/apache/datafusion-comet/pull/3739";>#3739</a>. These
classes were
-added upstream in <a class="reference external"
href="https://github.com/apache/iceberg/pull/15674";>apache/iceberg#15674</a>
and depend on Comet’s old Iceberg Java integration. Since Comet now
-uses a native Iceberg scan, these classes fail to compile and must be
removed.</p></li>
-<li><p>Configure test base classes (<code class="docutils literal
notranslate"><span class="pre">TestBase</span></code>, <code class="docutils
literal notranslate"><span class="pre">ExtensionsTestBase</span></code>, <code
class="docutils literal notranslate"><span
class="pre">ScanTestBase</span></code>, etc.) to load the Comet Spark
-plugin and shuffle manager</p></li>
+<li><p><strong>Prefix.</strong> All keys start with <code class="docutils
literal notranslate"><span class="pre">spark.comet.</span></code> — never bare
<code class="docutils literal notranslate"><span
class="pre">comet.</span></code> and never a nested
+prefix like <code class="docutils literal notranslate"><span
class="pre">spark.sql.comet.</span></code>.</p></li>
+<li><p><strong>Category segment.</strong> The first segment after the prefix
names a broad area:
+<code class="docutils literal notranslate"><span
class="pre">exec</span></code> (execution and expressions), <code
class="docutils literal notranslate"><span class="pre">scan</span></code>
(source readers), <code class="docutils literal notranslate"><span
class="pre">parquet</span></code> (Parquet-specific
+settings), <code class="docutils literal notranslate"><span
class="pre">shuffle</span></code> (shuffle behavior), <code class="docutils
literal notranslate"><span class="pre">columnar</span></code> (columnar
shuffle), <code class="docutils literal notranslate"><span
class="pre">explain</span></code> (plan
+explain/logging), <code class="docutils literal notranslate"><span
class="pre">metrics</span></code>, <code class="docutils literal
notranslate"><span class="pre">tracing</span></code>, <code class="docutils
literal notranslate"><span class="pre">testing</span></code>, <code
class="docutils literal notranslate"><span class="pre">convert</span></code>
(Spark → Comet source
+conversion), or <code class="docutils literal notranslate"><span
class="pre">expression</span></code> / <code class="docutils literal
notranslate"><span class="pre">operator</span></code> (per-expression /
per-operator flags).</p></li>
+<li><p><strong>Segment casing.</strong> Multi-word segments use <code
class="docutils literal notranslate"><span class="pre">camelCase</span></code>,
not dot- or kebab-separated.
+Prefer <code class="docutils literal notranslate"><span
class="pre">spark.comet.foo.maxThreadNum</span></code> over <code
class="docutils literal notranslate"><span
class="pre">spark.comet.foo.max.thread.num</span></code> or
+<code class="docutils literal notranslate"><span
class="pre">spark.comet.foo.max-thread-num</span></code>.</p></li>
+<li><p><strong>Boolean suffix.</strong> Descriptor-form boolean flags that
gate a subsystem or feature end
+in <code class="docutils literal notranslate"><span
class="pre">.enabled</span></code> — for example <code class="docutils literal
notranslate"><span class="pre">spark.comet.debug.enabled</span></code>, <code
class="docutils literal notranslate"><span
class="pre">spark.comet.metrics.enabled</span></code>,
+<code class="docutils literal notranslate"><span
class="pre">spark.comet.parquet.rowFilterPushdown.enabled</span></code>.
Action-form flags whose name is itself
+a verb (<code class="docutils literal notranslate"><span
class="pre">force...</span></code>, <code class="docutils literal
notranslate"><span class="pre">allow...</span></code>) can omit <code
class="docutils literal notranslate"><span class="pre">.enabled</span></code>
because the verb already encodes
+the action — for example <code class="docutils literal notranslate"><span
class="pre">spark.comet.exec.forceShuffledHashJoin</span></code>.</p></li>
+<li><p><strong>Acronyms.</strong> Treat acronyms consistently as <code
class="docutils literal notranslate"><span class="pre">UDF</span></code>, <code
class="docutils literal notranslate"><span class="pre">SHJ</span></code>, <code
class="docutils literal notranslate"><span class="pre">SMJ</span></code>, <code
class="docutils literal notranslate"><span class="pre">IO</span></code> —
either all-caps
+or camelCase, but do not mix within one key (<code class="docutils literal
notranslate"><span class="pre">pyarrowUdf</span></code> and <code
class="docutils literal notranslate"><span class="pre">scalaUDF</span></code>
in the same
+cluster is a red flag).</p></li>
</ul>
-<section id="install-comet">
-<h2>1. Install Comet<a class="headerlink" href="#install-comet" title="Link to
this heading">#</a></h2>
-<p>Run <code class="docutils literal notranslate"><span
class="pre">make</span> <span class="pre">release</span></code> in Comet to
install the Comet JAR into the local Maven repository, specifying the Spark
version.</p>
-<div class="highlight-shell notranslate"><div
class="highlight"><pre><span></span><span class="nv">PROFILES</span><span
class="o">=</span><span class="s2">"-Pspark-4.1"</span><span
class="w"> </span>make<span class="w"> </span>release
-</pre></div>
-</div>
</section>
-<section id="clone-iceberg-and-apply-diff">
-<h2>2. Clone Iceberg and Apply Diff<a class="headerlink"
href="#clone-iceberg-and-apply-diff" title="Link to this heading">#</a></h2>
-<p>Clone Apache Iceberg locally and apply the diff file from Comet against the
matching tag.</p>
-<div class="highlight-shell notranslate"><div
class="highlight"><pre><span></span>git<span class="w"> </span>clone<span
class="w"> </span>[email protected]:apache/iceberg.git<span class="w">
</span>apache-iceberg
-<span class="nb">cd</span><span class="w"> </span>apache-iceberg
-git<span class="w"> </span>checkout<span class="w"> </span>apache-iceberg-1.8.1
-git<span class="w"> </span>apply<span class="w">
</span>../datafusion-comet/dev/diffs/iceberg/1.8.1.diff
-</pre></div>
-</div>
-</section>
-<section id="run-iceberg-spark-tests">
-<h2>3. Run Iceberg Spark Tests<a class="headerlink"
href="#run-iceberg-spark-tests" title="Link to this heading">#</a></h2>
-<div class="highlight-shell notranslate"><div
class="highlight"><pre><span></span><span class="nv">ENABLE_COMET</span><span
class="o">=</span><span class="nb">true</span><span class="w">
</span>./gradlew<span class="w"> </span>-DsparkVersions<span
class="o">=</span><span class="m">3</span>.5<span class="w">
</span>-DscalaVersion<span class="o">=</span><span class="m">2</span>.13<span
class="w"> </span>-DflinkVersions<span class="o">=</span><span class="w">
</span>-DkafkaVersions<span cla [...]
-<span class="w"> </span>:iceberg-spark:iceberg-spark-3.5_2.13:test<span
class="w"> </span><span class="se">\</span>
-<span class="w"> </span>-Pquick<span class="o">=</span><span
class="nb">true</span><span class="w"> </span>-x<span class="w"> </span>javadoc
-</pre></div>
-</div>
-<p>The three Gradle targets tested in CI are:</p>
+<section id="symbol-naming-scala">
+<h2>Symbol Naming (Scala)<a class="headerlink" href="#symbol-naming-scala"
title="Link to this heading">#</a></h2>
+<p>Every config declares a Scala <code class="docutils literal
notranslate"><span class="pre">val</span></code> in <code class="docutils
literal notranslate"><span class="pre">CometConf.scala</span></code>. The
symbol name is the
+<code class="docutils literal notranslate"><span
class="pre">UPPER_SNAKE_CASE</span></code> form of the key’s meaningful
segments, prefixed with <code class="docutils literal notranslate"><span
class="pre">COMET_</span></code>.</p>
+<p>Examples:</p>
<div class="pst-scrollable-table-container"><table class="table">
<thead>
-<tr class="row-odd"><th class="head"><p>Gradle Target</p></th>
-<th class="head"><p>What It Covers</p></th>
+<tr class="row-odd"><th class="head"><p>Key</p></th>
+<th class="head"><p>Symbol</p></th>
</tr>
</thead>
<tbody>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span
class="pre">iceberg-spark-<ver>:test</span></code></p></td>
-<td><p>Core read/write paths (Parquet, Avro, ORC, vectorized), scan
operations, filtering, bloom filters, runtime filtering, deletion handling,
structured streaming, DDL/DML (create/alter/drop, writes, deletes), filter and
aggregate pushdown, actions (snapshot expiration, file rewriting, orphan
cleanup, table migration), serialization, and data format conversions.</p></td>
+<tr class="row-even"><td><p><code class="docutils literal notranslate"><span
class="pre">spark.comet.enabled</span></code></p></td>
+<td><p><code class="docutils literal notranslate"><span
class="pre">COMET_ENABLED</span></code></p></td>
</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span
class="pre">iceberg-spark-extensions-<ver>:test</span></code></p></td>
-<td><p>SQL extensions: stored procedures (migrate, snapshot, cherrypick,
rollback, rewrite-data-files, rewrite-manifests, expire-snapshots,
remove-orphan-files, etc.), row-level operations (copy-on-write and
merge-on-read update/delete/merge), DDL extensions (branches, tags, alter
schema, partition fields), changelog tables/views, metadata tables, and
views.</p></td>
+<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span
class="pre">spark.comet.exec.forceShuffledHashJoin</span></code></p></td>
+<td><p><code class="docutils literal notranslate"><span
class="pre">COMET_FORCE_SHJ</span></code></p></td>
</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span
class="pre">iceberg-spark-runtime-<ver>:integrationTest</span></code></p></td>
-<td><p>A single smoke test (<code class="docutils literal notranslate"><span
class="pre">SmokeTest.java</span></code>) that validates the shaded runtime
JAR. The <code class="docutils literal notranslate"><span
class="pre">spark-runtime</span></code> module has no main source — it packages
Iceberg and all dependencies into a shaded uber-JAR. The smoke test exercises
basic create, insert, merge, query, partition field, and sort order operations
to confirm the shaded JAR works end-to-end.< [...]
+<tr class="row-even"><td><p><code class="docutils literal notranslate"><span
class="pre">spark.comet.parquet.rowFilterPushdown.enabled</span></code></p></td>
+<td><p><code class="docutils literal notranslate"><span
class="pre">COMET_ROW_FILTER_PUSHDOWN</span></code></p></td>
</tr>
</tbody>
</table>
</div>
+<p>The symbol name is what appears in code; the key is what appears in user
configuration.
+The two do not have to match segment-for-segment — brevity in the symbol is
fine as long
+as the key remains descriptive.</p>
+</section>
+<section id="categories">
+<h2>Categories<a class="headerlink" href="#categories" title="Link to this
heading">#</a></h2>
+<p>Every <code class="docutils literal notranslate"><span
class="pre">ConfigEntry</span></code> must call <code class="docutils literal
notranslate"><span class="pre">.category(...)</span></code>. The category is
used to route the key into
+the right table in the user guide’s <code class="docutils literal
notranslate"><span class="pre">configs.md</span></code>. Available categories
are declared as
+<code class="docutils literal notranslate"><span
class="pre">CATEGORY_*</span></code> constants at the top of <code
class="docutils literal notranslate"><span
class="pre">CometConf.scala</span></code>. If a new config does not fit an
+existing category, discuss adding a new one before landing the config.</p>
</section>
-<section id="updating-diffs">
-<h2>Updating Diffs<a class="headerlink" href="#updating-diffs" title="Link to
this heading">#</a></h2>
-<p>To update a diff (e.g. after modifying test configuration), apply the
existing diff, make changes, then
-regenerate:</p>
-<div class="highlight-shell notranslate"><div
class="highlight"><pre><span></span><span class="nb">cd</span><span class="w">
</span>apache-iceberg
-git<span class="w"> </span>reset<span class="w"> </span>--hard<span class="w">
</span>apache-iceberg-1.8.1<span class="w"> </span><span
class="o">&&</span><span class="w"> </span>git<span class="w">
</span>clean<span class="w"> </span>-fd
-git<span class="w"> </span>apply<span class="w">
</span>../datafusion-comet/dev/diffs/iceberg/1.8.1.diff
-
-<span class="c1"># Make changes, then run spotless to fix formatting</span>
-./gradlew<span class="w"> </span>spotlessApply
-
-<span class="c1"># Stage any new or deleted files, then generate the
diff</span>
-git<span class="w"> </span>add<span class="w"> </span>-A
-git<span class="w"> </span>diff<span class="w">
</span>apache-iceberg-1.8.1<span class="w"> </span>><span class="w">
</span>../datafusion-comet/dev/diffs/iceberg/1.8.1.diff
+<section id="renaming-an-existing-config">
+<h2>Renaming an Existing Config<a class="headerlink"
href="#renaming-an-existing-config" title="Link to this heading">#</a></h2>
+<p>Configs under <code class="docutils literal notranslate"><span
class="pre">spark.comet.*</span></code> are stable across minor releases: users
may have set them in
+production <code class="docutils literal notranslate"><span
class="pre">spark-defaults.conf</span></code> files, Spark job submissions, or
notebooks. Renaming a key
+must not silently break those deployments.</p>
+<p>Use the <code class="docutils literal notranslate"><span
class="pre">withAlternative</span></code> builder on <code class="docutils
literal notranslate"><span class="pre">ConfigBuilder</span></code> to keep the
old key working as a
+deprecated alias:</p>
+<div class="highlight-scala notranslate"><div
class="highlight"><pre><span></span><span class="kd">val</span><span class="w">
</span><span class="nc">COMET_FORCE_SHJ</span><span class="p">:</span><span
class="w"> </span><span class="nc">ConfigEntry</span><span
class="p">[</span><span class="nc">Boolean</span><span class="p">]</span><span
class="w"> </span><span class="o">=</span>
+<span class="w"> </span><span class="n">conf</span><span
class="p">(</span><span class="s">s"</span><span class="si">$</span><span
class="n">COMET_EXEC_CONFIG_PREFIX</span><span
class="s">.forceShuffledHashJoin"</span><span class="p">)</span>
+<span class="w"> </span><span class="p">.</span><span
class="n">withAlternative</span><span class="p">(</span><span
class="s">s"</span><span class="si">$</span><span
class="n">COMET_EXEC_CONFIG_PREFIX</span><span
class="s">.replaceSortMergeJoin"</span><span class="p">)</span>
+<span class="w"> </span><span class="p">.</span><span
class="n">category</span><span class="p">(</span><span
class="nc">CATEGORY_EXEC</span><span class="p">)</span>
+<span class="w"> </span><span class="p">.</span><span
class="n">doc</span><span class="p">(</span><span
class="s">"..."</span><span class="p">)</span>
+<span class="w"> </span><span class="p">.</span><span
class="n">booleanConf</span>
+<span class="w"> </span><span class="p">.</span><span
class="n">createWithDefault</span><span class="p">(</span><span
class="kc">false</span><span class="p">)</span>
</pre></div>
</div>
-<p>Repeat for each Iceberg version (1.8.1, 1.9.1, 1.10.0, 1.11.0). The file
contents differ between versions, so each
-diff must be generated against its own tag.</p>
-</section>
-<section id="running-tests-in-ci">
-<h2>Running Tests in CI<a class="headerlink" href="#running-tests-in-ci"
title="Link to this heading">#</a></h2>
-<p>The <code class="docutils literal notranslate"><span
class="pre">iceberg_spark_test_<version>.yml</span></code> workflows
apply these diffs and run the three Gradle targets above
-against each Iceberg version. Iceberg 1.8.1 runs against Spark 3.4.3 with Java
11; Iceberg 1.9.1 and 1.10.0
-run against Spark 3.5.8 with Java 17; Iceberg 1.11.0 runs against Spark 4.1.2
with Java 17. Iceberg 1.11
-(the only version testing Spark 4.1) runs on every pull request and on pushes
to main; the older versions
-(1.8, 1.9, 1.10) run only on pushes to main, or on a pull request labeled
<code class="docutils literal notranslate"><span
class="pre">run-iceberg-tests</span></code>. All caller
-workflows delegate to <code class="docutils literal notranslate"><span
class="pre">iceberg_spark_test_reusable.yml</span></code>, which holds the
build and test job logic.</p>
+<p>Reading a value from the alternative logs a one-time deprecation warning
per JVM per
+alternative key, pointing users at the current key. The primary key always
wins if both
+are set.</p>
+<p><code class="docutils literal notranslate"><span
class="pre">withAlternative</span></code> accepts multiple alternatives
(checked in order), for keys that have
+been renamed more than once.</p>
+<p>The rename checklist for a single config:</p>
+<ol class="arabic simple">
+<li><p>Update the key string on the <code class="docutils literal
notranslate"><span class="pre">conf(...)</span></code> call and add <code
class="docutils literal notranslate"><span
class="pre">.withAlternative(oldKey)</span></code>.</p></li>
+<li><p>Rename the Scala <code class="docutils literal notranslate"><span
class="pre">val</span></code> (for example <code class="docutils literal
notranslate"><span class="pre">COMET_REPLACE_SMJ</span></code> → <code
class="docutils literal notranslate"><span
class="pre">COMET_FORCE_SHJ</span></code>).</p></li>
+<li><p>Update every callsite of the Scala symbol — grep the whole
repo.</p></li>
+<li><p>Update every documented reference to the old key string — grep <code
class="docutils literal notranslate"><span
class="pre">docs/source/</span></code> for the
+old key and update to the new key. The auto-generated <code class="docutils
literal notranslate"><span class="pre">configs.md</span></code> table will
refresh
+on the next release-docs regeneration and does not need manual
editing.</p></li>
+<li><p>Add or update a test in <code class="docutils literal
notranslate"><span class="pre">CometConfSuite</span></code> if the rename
covers a new type of alias
+pattern (single-hop, multiple alternatives, etc.).</p></li>
+</ol>
+<p>Removing a deprecated alias is a follow-up step that belongs to a later
major release —
+typically the next Comet major after the rename first ships. See the
+<a class="reference internal" href="../about/versioning_policy.html"><span
class="std std-doc">versioning policy</span></a> for the timing rules.</p>
</section>
</section>
@@ -643,20 +660,20 @@ workflows delegate to <code class="docutils literal
notranslate"><span class="pr
<div class="prev-next-area">
<a class="left-prev"
- href="spark-sql-tests.html"
+ href="spark_configs_support.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Running Spark SQL Tests</p>
+ <p class="prev-next-title">Supported Spark Configurations</p>
</div>
</a>
<a class="right-next"
- href="debugging.html"
+ href="bug_triage.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Comet Debugging Guide</p>
+ <p class="prev-next-title">Bug Triage Guide</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
@@ -679,11 +696,10 @@ workflows delegate to <code class="docutils literal
notranslate"><span class="pr
</div>
<nav class="bd-toc-nav page-toc"
aria-labelledby="pst-page-navigation-heading-2">
<ul class="visible nav section-nav flex-column">
-<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link"
href="#install-comet">1. Install Comet</a></li>
-<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link"
href="#clone-iceberg-and-apply-diff">2. Clone Iceberg and Apply Diff</a></li>
-<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link"
href="#run-iceberg-spark-tests">3. Run Iceberg Spark Tests</a></li>
-<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link"
href="#updating-diffs">Updating Diffs</a></li>
-<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link"
href="#running-tests-in-ci">Running Tests in CI</a></li>
+<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link"
href="#key-naming">Key Naming</a></li>
+<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link"
href="#symbol-naming-scala">Symbol Naming (Scala)</a></li>
+<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link"
href="#categories">Categories</a></li>
+<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link"
href="#renaming-an-existing-config">Renaming an Existing Config</a></li>
</ul>
</nav></div>
diff --git a/contributor-guide/contributing.html
b/contributor-guide/contributing.html
index 47499b162d..779d0c946c 100644
--- a/contributor-guide/contributing.html
+++ b/contributor-guide/contributing.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/debugging.html b/contributor-guide/debugging.html
index a52444c943..86ff318ff2 100644
--- a/contributor-guide/debugging.html
+++ b/contributor-guide/debugging.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/development.html
b/contributor-guide/development.html
index 5addbbc499..971e6215f3 100644
--- a/contributor-guide/development.html
+++ b/contributor-guide/development.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/agg_funcs.html
b/contributor-guide/expression-audits/agg_funcs.html
index 445f2e7072..493f686f02 100644
--- a/contributor-guide/expression-audits/agg_funcs.html
+++ b/contributor-guide/expression-audits/agg_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/array_funcs.html
b/contributor-guide/expression-audits/array_funcs.html
index 306c25af3b..439f939664 100644
--- a/contributor-guide/expression-audits/array_funcs.html
+++ b/contributor-guide/expression-audits/array_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/bitwise_funcs.html
b/contributor-guide/expression-audits/bitwise_funcs.html
index 1eb4fdc286..f0c9ae86df 100644
--- a/contributor-guide/expression-audits/bitwise_funcs.html
+++ b/contributor-guide/expression-audits/bitwise_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/collection_funcs.html
b/contributor-guide/expression-audits/collection_funcs.html
index 0bc969ec44..d20d067b1e 100644
--- a/contributor-guide/expression-audits/collection_funcs.html
+++ b/contributor-guide/expression-audits/collection_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/conditional_funcs.html
b/contributor-guide/expression-audits/conditional_funcs.html
index 731c48fe44..63cc842836 100644
--- a/contributor-guide/expression-audits/conditional_funcs.html
+++ b/contributor-guide/expression-audits/conditional_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/conversion_funcs.html
b/contributor-guide/expression-audits/conversion_funcs.html
index ba295fd238..2fb8948e34 100644
--- a/contributor-guide/expression-audits/conversion_funcs.html
+++ b/contributor-guide/expression-audits/conversion_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/datetime_funcs.html
b/contributor-guide/expression-audits/datetime_funcs.html
index d973d581df..6783ad19bd 100644
--- a/contributor-guide/expression-audits/datetime_funcs.html
+++ b/contributor-guide/expression-audits/datetime_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/generator_funcs.html
b/contributor-guide/expression-audits/generator_funcs.html
index 9c9eafb9fd..58c1d7d40c 100644
--- a/contributor-guide/expression-audits/generator_funcs.html
+++ b/contributor-guide/expression-audits/generator_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/hash_funcs.html
b/contributor-guide/expression-audits/hash_funcs.html
index d7586f1e64..584e996403 100644
--- a/contributor-guide/expression-audits/hash_funcs.html
+++ b/contributor-guide/expression-audits/hash_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/index.html
b/contributor-guide/expression-audits/index.html
index a58af136fc..e33e5a3834 100644
--- a/contributor-guide/expression-audits/index.html
+++ b/contributor-guide/expression-audits/index.html
@@ -452,6 +452,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/json_funcs.html
b/contributor-guide/expression-audits/json_funcs.html
index d87fde42ae..2c2f86f074 100644
--- a/contributor-guide/expression-audits/json_funcs.html
+++ b/contributor-guide/expression-audits/json_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/map_funcs.html
b/contributor-guide/expression-audits/map_funcs.html
index 3e6c6c5997..d1a2dacc2f 100644
--- a/contributor-guide/expression-audits/map_funcs.html
+++ b/contributor-guide/expression-audits/map_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/math_funcs.html
b/contributor-guide/expression-audits/math_funcs.html
index a59bd4efe8..0e68e9d7cf 100644
--- a/contributor-guide/expression-audits/math_funcs.html
+++ b/contributor-guide/expression-audits/math_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/misc_funcs.html
b/contributor-guide/expression-audits/misc_funcs.html
index 88fc0dbe6f..4f3aecf077 100644
--- a/contributor-guide/expression-audits/misc_funcs.html
+++ b/contributor-guide/expression-audits/misc_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/predicate_funcs.html
b/contributor-guide/expression-audits/predicate_funcs.html
index 117aa9fe5b..df470ecb4d 100644
--- a/contributor-guide/expression-audits/predicate_funcs.html
+++ b/contributor-guide/expression-audits/predicate_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/string_funcs.html
b/contributor-guide/expression-audits/string_funcs.html
index a093d3a5ae..8d55ecac10 100644
--- a/contributor-guide/expression-audits/string_funcs.html
+++ b/contributor-guide/expression-audits/string_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/struct_funcs.html
b/contributor-guide/expression-audits/struct_funcs.html
index f0e66590fb..76a97730ba 100644
--- a/contributor-guide/expression-audits/struct_funcs.html
+++ b/contributor-guide/expression-audits/struct_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/url_funcs.html
b/contributor-guide/expression-audits/url_funcs.html
index da33fa0565..21037ef940 100644
--- a/contributor-guide/expression-audits/url_funcs.html
+++ b/contributor-guide/expression-audits/url_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/expression-audits/window_funcs.html
b/contributor-guide/expression-audits/window_funcs.html
index ea6b6ad302..05090c0841 100644
--- a/contributor-guide/expression-audits/window_funcs.html
+++ b/contributor-guide/expression-audits/window_funcs.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="../spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="../config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/ffi.html b/contributor-guide/ffi.html
index 0a5f5d364d..86c702bf8a 100644
--- a/contributor-guide/ffi.html
+++ b/contributor-guide/ffi.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/iceberg-spark-tests.html
b/contributor-guide/iceberg-spark-tests.html
index 03d14f67e6..1a8981a017 100644
--- a/contributor-guide/iceberg-spark-tests.html
+++ b/contributor-guide/iceberg-spark-tests.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/index.html b/contributor-guide/index.html
index 0a20afbdf0..366b1f1f30 100644
--- a/contributor-guide/index.html
+++ b/contributor-guide/index.html
@@ -452,6 +452,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/jvm_shuffle.html
b/contributor-guide/jvm_shuffle.html
index 8f453fbda7..b4238f24e5 100644
--- a/contributor-guide/jvm_shuffle.html
+++ b/contributor-guide/jvm_shuffle.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/native_shuffle.html
b/contributor-guide/native_shuffle.html
index c4007c137d..9666f40a41 100644
--- a/contributor-guide/native_shuffle.html
+++ b/contributor-guide/native_shuffle.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/optimizing_expressions.html
b/contributor-guide/optimizing_expressions.html
index 1b4e4ea2d3..fa08c1de0d 100644
--- a/contributor-guide/optimizing_expressions.html
+++ b/contributor-guide/optimizing_expressions.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/plugin_overview.html
b/contributor-guide/plugin_overview.html
index 7493850569..7fd93fda81 100644
--- a/contributor-guide/plugin_overview.html
+++ b/contributor-guide/plugin_overview.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/profiling.html b/contributor-guide/profiling.html
index 4d86473050..21a167aac1 100644
--- a/contributor-guide/profiling.html
+++ b/contributor-guide/profiling.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/release_process.html
b/contributor-guide/release_process.html
index 48269ff53b..234d64c964 100644
--- a/contributor-guide/release_process.html
+++ b/contributor-guide/release_process.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="current nav bd-sidenav">
diff --git a/contributor-guide/roadmap.html b/contributor-guide/roadmap.html
index 6b5c0631f6..1931e5aa9c 100644
--- a/contributor-guide/roadmap.html
+++ b/contributor-guide/roadmap.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="current nav bd-sidenav">
diff --git a/contributor-guide/s3-credential-provider-design.html
b/contributor-guide/s3-credential-provider-design.html
index 4c129fa96a..b5bb2e6017 100644
--- a/contributor-guide/s3-credential-provider-design.html
+++ b/contributor-guide/s3-credential-provider-design.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/spark-sql-tests.html
b/contributor-guide/spark-sql-tests.html
index 138cff09bc..434c2ec0d9 100644
--- a/contributor-guide/spark-sql-tests.html
+++ b/contributor-guide/spark-sql-tests.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/spark_configs_support.html
b/contributor-guide/spark_configs_support.html
index eba31247de..d1c9474a60 100644
--- a/contributor-guide/spark_configs_support.html
+++ b/contributor-guide/spark_configs_support.html
@@ -68,7 +68,7 @@ under the License.
<script src="../_static/comet-ux.js?v=97577a30"></script>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
- <link rel="next" title="Bug Triage Guide" href="bug_triage.html" />
+ <link rel="next" title="Configuration Conventions"
href="config_conventions.html" />
<link rel="prev" title="window_funcs Expression Audits"
href="expression-audits/window_funcs.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1 current active"><a class="current reference internal"
href="#">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
@@ -676,11 +677,11 @@ the Comet pipeline. Tests for expressions with known
fallback paths continue to
</div>
</a>
<a class="right-next"
- href="bug_triage.html"
+ href="config_conventions.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">Bug Triage Guide</p>
+ <p class="prev-next-title">Configuration Conventions</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
diff --git a/contributor-guide/sql-file-tests.html
b/contributor-guide/sql-file-tests.html
index b972f2e700..976c5ade15 100644
--- a/contributor-guide/sql-file-tests.html
+++ b/contributor-guide/sql-file-tests.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/sql_error_propagation.html
b/contributor-guide/sql_error_propagation.html
index d08b080992..67591d2822 100644
--- a/contributor-guide/sql_error_propagation.html
+++ b/contributor-guide/sql_error_propagation.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/contributor-guide/tracing.html b/contributor-guide/tracing.html
index f7a2cddac7..836a84a838 100644
--- a/contributor-guide/tracing.html
+++ b/contributor-guide/tracing.html
@@ -456,6 +456,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal"
href="spark_configs_support.html">Supported Spark Configurations</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="config_conventions.html">Configuration Conventions</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span
class="caption-text">Project Mechanics</span></p>
<ul class="nav bd-sidenav">
diff --git a/genindex.html b/genindex.html
index 938a6dc7f3..4c50d05048 100644
--- a/genindex.html
+++ b/genindex.html
@@ -567,6 +567,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal"
href="contributor-guide/spark_configs_support.html">Supported Spark
Configurations</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="contributor-guide/config_conventions.html">Configuration
Conventions</a></li>
<li class="toctree-l2"><a class="reference internal"
href="contributor-guide/bug_triage.html">Bug Triage</a></li>
<li class="toctree-l2"><a class="reference internal"
href="contributor-guide/release_process.html">Release Process</a></li>
<li class="toctree-l2"><a class="reference internal"
href="contributor-guide/roadmap.html">Roadmap</a></li>
diff --git a/objects.inv b/objects.inv
index 59331ea05a..55df9b4134 100644
Binary files a/objects.inv and b/objects.inv differ
diff --git a/search.html b/search.html
index f4922c582c..b264a62fae 100644
--- a/search.html
+++ b/search.html
@@ -569,6 +569,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal"
href="contributor-guide/spark_configs_support.html">Supported Spark
Configurations</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="contributor-guide/config_conventions.html">Configuration
Conventions</a></li>
<li class="toctree-l2"><a class="reference internal"
href="contributor-guide/bug_triage.html">Bug Triage</a></li>
<li class="toctree-l2"><a class="reference internal"
href="contributor-guide/release_process.html">Release Process</a></li>
<li class="toctree-l2"><a class="reference internal"
href="contributor-guide/roadmap.html">Roadmap</a></li>
diff --git a/searchindex.js b/searchindex.js
index b15a7da6c9..75694070db 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles": {"!": [[52, "id1"]], "%": [[50, "id1"]], "&":
[[40, "id1"]], "*": [[50, "id2"]], "+": [[50, "id3"]], "-": [[50, "id4"]], "/":
[[50, "id5"]], "1. Format Your Code": [[37, "format-your-code"]], "1. Install
Comet": [[58, "install-comet"], [68, "install-comet"]], "1. Native Operators
(nativeExecs map)": [[26, "native-operators-nativeexecs-map"]], "2. Build and
Verify": [[37, "build-and-verify"]], "2. Clone Iceberg and Apply Diff": [[58,
"clone-iceberg-and-apply- [...]
\ No newline at end of file
+Search.setIndex({"alltitles": {"!": [[53, "id1"]], "%": [[51, "id1"]], "&":
[[41, "id1"]], "*": [[51, "id2"]], "+": [[51, "id3"]], "-": [[51, "id4"]], "/":
[[51, "id5"]], "1. Format Your Code": [[38, "format-your-code"]], "1. Install
Comet": [[59, "install-comet"], [69, "install-comet"]], "1. Native Operators
(nativeExecs map)": [[26, "native-operators-nativeexecs-map"]], "2. Build and
Verify": [[38, "build-and-verify"]], "2. Clone Iceberg and Apply Diff": [[59,
"clone-iceberg-and-apply- [...]
\ No newline at end of file
diff --git
a/user-guide/0.17/compatibility/expressions/_category_template/aggregate.html
b/user-guide/0.17/compatibility/expressions/_category_template/aggregate.html
index fc7d896283..c84ca3af42 100644
---
a/user-guide/0.17/compatibility/expressions/_category_template/aggregate.html
+++
b/user-guide/0.17/compatibility/expressions/_category_template/aggregate.html
@@ -568,6 +568,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/spark_configs_support.html">Supported
Spark Configurations</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/config_conventions.html">Configuration
Conventions</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/bug_triage.html">Bug Triage</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/release_process.html">Release
Process</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/roadmap.html">Roadmap</a></li>
diff --git
a/user-guide/0.17/compatibility/expressions/_category_template/array.html
b/user-guide/0.17/compatibility/expressions/_category_template/array.html
index de81429b85..2c0aca8c06 100644
--- a/user-guide/0.17/compatibility/expressions/_category_template/array.html
+++ b/user-guide/0.17/compatibility/expressions/_category_template/array.html
@@ -568,6 +568,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/spark_configs_support.html">Supported
Spark Configurations</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/config_conventions.html">Configuration
Conventions</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/bug_triage.html">Bug Triage</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/release_process.html">Release
Process</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/roadmap.html">Roadmap</a></li>
diff --git
a/user-guide/0.17/compatibility/expressions/_category_template/cast.html
b/user-guide/0.17/compatibility/expressions/_category_template/cast.html
index 3cbe77487d..5d16e02636 100644
--- a/user-guide/0.17/compatibility/expressions/_category_template/cast.html
+++ b/user-guide/0.17/compatibility/expressions/_category_template/cast.html
@@ -572,6 +572,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/spark_configs_support.html">Supported
Spark Configurations</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/config_conventions.html">Configuration
Conventions</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/bug_triage.html">Bug Triage</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/release_process.html">Release
Process</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/roadmap.html">Roadmap</a></li>
diff --git
a/user-guide/0.17/compatibility/expressions/_category_template/datetime.html
b/user-guide/0.17/compatibility/expressions/_category_template/datetime.html
index 0e7ff6149d..6fcd2d67f5 100644
--- a/user-guide/0.17/compatibility/expressions/_category_template/datetime.html
+++ b/user-guide/0.17/compatibility/expressions/_category_template/datetime.html
@@ -572,6 +572,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/spark_configs_support.html">Supported
Spark Configurations</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/config_conventions.html">Configuration
Conventions</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/bug_triage.html">Bug Triage</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/release_process.html">Release
Process</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/roadmap.html">Roadmap</a></li>
diff --git
a/user-guide/0.17/compatibility/expressions/_category_template/map.html
b/user-guide/0.17/compatibility/expressions/_category_template/map.html
index 0a40d74125..aca5162013 100644
--- a/user-guide/0.17/compatibility/expressions/_category_template/map.html
+++ b/user-guide/0.17/compatibility/expressions/_category_template/map.html
@@ -572,6 +572,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/spark_configs_support.html">Supported
Spark Configurations</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/config_conventions.html">Configuration
Conventions</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/bug_triage.html">Bug Triage</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/release_process.html">Release
Process</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/roadmap.html">Roadmap</a></li>
diff --git
a/user-guide/0.17/compatibility/expressions/_category_template/math.html
b/user-guide/0.17/compatibility/expressions/_category_template/math.html
index 78090061fa..73b2d66ba7 100644
--- a/user-guide/0.17/compatibility/expressions/_category_template/math.html
+++ b/user-guide/0.17/compatibility/expressions/_category_template/math.html
@@ -568,6 +568,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/spark_configs_support.html">Supported
Spark Configurations</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/config_conventions.html">Configuration
Conventions</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/bug_triage.html">Bug Triage</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/release_process.html">Release
Process</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/roadmap.html">Roadmap</a></li>
diff --git
a/user-guide/0.17/compatibility/expressions/_category_template/misc.html
b/user-guide/0.17/compatibility/expressions/_category_template/misc.html
index 4870cdda12..5158e73e25 100644
--- a/user-guide/0.17/compatibility/expressions/_category_template/misc.html
+++ b/user-guide/0.17/compatibility/expressions/_category_template/misc.html
@@ -568,6 +568,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/spark_configs_support.html">Supported
Spark Configurations</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/config_conventions.html">Configuration
Conventions</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/bug_triage.html">Bug Triage</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/release_process.html">Release
Process</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/roadmap.html">Roadmap</a></li>
diff --git
a/user-guide/0.17/compatibility/expressions/_category_template/string.html
b/user-guide/0.17/compatibility/expressions/_category_template/string.html
index 2ab2597297..38b3e2b34c 100644
--- a/user-guide/0.17/compatibility/expressions/_category_template/string.html
+++ b/user-guide/0.17/compatibility/expressions/_category_template/string.html
@@ -568,6 +568,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/spark_configs_support.html">Supported
Spark Configurations</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/config_conventions.html">Configuration
Conventions</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/bug_triage.html">Bug Triage</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/release_process.html">Release
Process</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/roadmap.html">Roadmap</a></li>
diff --git
a/user-guide/0.17/compatibility/expressions/_category_template/struct.html
b/user-guide/0.17/compatibility/expressions/_category_template/struct.html
index 551b7b30bb..f1610911de 100644
--- a/user-guide/0.17/compatibility/expressions/_category_template/struct.html
+++ b/user-guide/0.17/compatibility/expressions/_category_template/struct.html
@@ -568,6 +568,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/spark_configs_support.html">Supported
Spark Configurations</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/config_conventions.html">Configuration
Conventions</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/bug_triage.html">Bug Triage</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/release_process.html">Release
Process</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/roadmap.html">Roadmap</a></li>
diff --git
a/user-guide/0.17/compatibility/expressions/_category_template/url.html
b/user-guide/0.17/compatibility/expressions/_category_template/url.html
index e5ef64f2ca..7cba9cf5e0 100644
--- a/user-guide/0.17/compatibility/expressions/_category_template/url.html
+++ b/user-guide/0.17/compatibility/expressions/_category_template/url.html
@@ -568,6 +568,7 @@ under the License.
</ul>
</details></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/spark_configs_support.html">Supported
Spark Configurations</a></li>
+<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/config_conventions.html">Configuration
Conventions</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/bug_triage.html">Bug Triage</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/release_process.html">Release
Process</a></li>
<li class="toctree-l2"><a class="reference internal"
href="../../../../../contributor-guide/roadmap.html">Roadmap</a></li>
diff --git a/user-guide/latest/configs.html b/user-guide/latest/configs.html
index 8f3732665b..f8df7199f1 100644
--- a/user-guide/latest/configs.html
+++ b/user-guide/latest/configs.html
@@ -850,12 +850,12 @@ under the License.
<td><p>Whether to enable Comet native vectorized execution for Spark. This
controls whether Spark should convert operators into their Comet counterparts
and execute them in native space. Note: each operator is associated with a
separate config in the format of <code class="docutils literal
notranslate"><span
class="pre">spark.comet.exec.<operator_name>.enabled</span></code> at the
moment, and both the config and this need to be turned on, in order for the
operator to be executed in [...]
<td><p>true</p></td>
</tr>
-<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span
class="pre">spark.comet.exec.pyarrowUdf.enabled</span></code></p></td>
-<td><p>Experimental: whether to enable optimized execution of PyArrow UDFs
(mapInArrow/mapInPandas). When enabled, Comet passes Arrow columnar data
directly to Python UDFs without the intermediate Arrow-to-Row-to-Arrow
conversion that Spark normally performs. Disabled by default while the feature
stabilizes.</p></td>
+<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span
class="pre">spark.comet.exec.forceShuffledHashJoin</span></code></p></td>
+<td><p>Experimental feature to force Spark to replace SortMergeJoin with
ShuffledHashJoin for improved performance. This feature is not stable yet. For
more information, refer to the <a class="reference external"
href="https://datafusion.apache.org/comet/user-guide/latest/tuning.html";>Comet
Tuning Guide</a>.</p></td>
<td><p>false</p></td>
</tr>
-<tr class="row-even"><td><p><code class="docutils literal notranslate"><span
class="pre">spark.comet.exec.replaceSortMergeJoin</span></code></p></td>
-<td><p>Experimental feature to force Spark to replace SortMergeJoin with
ShuffledHashJoin for improved performance. This feature is not stable yet. For
more information, refer to the <a class="reference external"
href="https://datafusion.apache.org/comet/user-guide/latest/tuning.html";>Comet
Tuning Guide</a>.</p></td>
+<tr class="row-even"><td><p><code class="docutils literal notranslate"><span
class="pre">spark.comet.exec.pyarrowUdf.enabled</span></code></p></td>
+<td><p>Experimental: whether to enable optimized execution of PyArrow UDFs
(mapInArrow/mapInPandas). When enabled, Comet passes Arrow columnar data
directly to Python UDFs without the intermediate Arrow-to-Row-to-Arrow
conversion that Spark normally performs. Disabled by default while the feature
stabilizes.</p></td>
<td><p>false</p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span
class="pre">spark.comet.exec.scalaUDF.codegen.enabled</span></code></p></td>
diff --git a/user-guide/latest/tuning.html b/user-guide/latest/tuning.html
index 0a2b94488a..502c788c88 100644
--- a/user-guide/latest/tuning.html
+++ b/user-guide/latest/tuning.html
@@ -822,7 +822,7 @@ and sorting on floating-point data can be enabled by
setting <code class="docuti
to configure Comet to convert <code class="docutils literal notranslate"><span
class="pre">SortMergeJoin</span></code> to <code class="docutils literal
notranslate"><span class="pre">ShuffledHashJoin</span></code>. Comet does not
yet provide spill-to-disk for
<code class="docutils literal notranslate"><span
class="pre">ShuffledHashJoin</span></code> so this could result in OOM. Also,
<code class="docutils literal notranslate"><span
class="pre">SortMergeJoin</span></code> may still be faster in some cases. It
is best
to test with both for your specific workloads.</p>
-<p>To configure Comet to convert <code class="docutils literal
notranslate"><span class="pre">SortMergeJoin</span></code> to <code
class="docutils literal notranslate"><span
class="pre">ShuffledHashJoin</span></code>, set <code class="docutils literal
notranslate"><span
class="pre">spark.comet.exec.replaceSortMergeJoin=true</span></code>.</p>
+<p>To configure Comet to convert <code class="docutils literal
notranslate"><span class="pre">SortMergeJoin</span></code> to <code
class="docutils literal notranslate"><span
class="pre">ShuffledHashJoin</span></code>, set <code class="docutils literal
notranslate"><span
class="pre">spark.comet.exec.forceShuffledHashJoin=true</span></code>.</p>
</section>
<section id="shuffle">
<h2>Shuffle<a class="headerlink" href="#shuffle" title="Link to this
heading">#</a></h2>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
