This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 63cfb1def docs: Update configuration guide to show optional configs
(#1524)
63cfb1def is described below
commit 63cfb1defebadf10a1ae1a71c59570be2117af2a
Author: Andy Grove <[email protected]>
AuthorDate: Thu Mar 13 16:51:10 2025 -0600
docs: Update configuration guide to show optional configs (#1524)
---
common/src/main/scala/org/apache/comet/CometConf.scala | 6 ++++--
docs/source/user-guide/configs.md | 1 +
spark/src/main/scala/org/apache/comet/GenerateDocs.scala | 6 +++++-
3 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/common/src/main/scala/org/apache/comet/CometConf.scala
b/common/src/main/scala/org/apache/comet/CometConf.scala
index 532e43f13..33039b744 100644
--- a/common/src/main/scala/org/apache/comet/CometConf.scala
+++ b/common/src/main/scala/org/apache/comet/CometConf.scala
@@ -54,7 +54,7 @@ object CometConf extends ShimCometConf {
/** List of all configs that is used for generating documentation */
val allConfs = new ListBuffer[ConfigEntry[_]]
- def register(conf: ConfigEntryWithDefault[_]): Unit = {
+ def register(conf: ConfigEntry[_]): Unit = {
allConfs.append(conf)
}
@@ -746,13 +746,15 @@ private class TypedConfigBuilder[T](
/** Creates a [[ConfigEntry]] that does not have a default value. */
def createOptional: OptionalConfigEntry[T] = {
- new OptionalConfigEntry[T](
+ val conf = new OptionalConfigEntry[T](
parent.key,
converter,
stringConverter,
parent._doc,
parent._public,
parent._version)
+ CometConf.register(conf)
+ conf
}
/** Creates a [[ConfigEntry]] that has a default value. */
diff --git a/docs/source/user-guide/configs.md
b/docs/source/user-guide/configs.md
index 33dc393b7..a208d211e 100644
--- a/docs/source/user-guide/configs.md
+++ b/docs/source/user-guide/configs.md
@@ -73,6 +73,7 @@ Comet provides the following configuration settings.
| spark.comet.expression.allowIncompatible | Comet is not currently fully
compatible with Spark for all expressions. Set this config to true to allow
them anyway. For more information, refer to the Comet Compatibility Guide
(https://datafusion.apache.org/comet/user-guide/compatibility.html). | false |
| spark.comet.memory.overhead.factor | Fraction of executor memory to be
allocated as additional non-heap memory per executor process for Comet. | 0.2 |
| spark.comet.memory.overhead.min | Minimum amount of additional memory to be
allocated per executor process for Comet, in MiB. | 402653184b |
+| spark.comet.memoryOverhead | The amount of additional memory to be allocated
per executor process for Comet, in MiB. This config is optional. If this is not
specified, it will be set to `spark.comet.memory.overhead.factor` *
`spark.executor.memory`. This is memory that accounts for things like Comet
native execution, Comet shuffle, etc. | |
| spark.comet.metrics.updateInterval | The interval in milliseconds to update
metrics. If interval is negative, metrics will be updated upon task completion.
| 3000 |
| spark.comet.nativeLoadRequired | Whether to require Comet native library to
load successfully when Comet is enabled. If not, Comet will silently fallback
to Spark when it fails to load the native lib. Otherwise, an error will be
thrown and the Spark job will be aborted. | false |
| spark.comet.parquet.enable.directBuffer | Whether to use Java direct byte
buffer when reading Parquet. | false |
diff --git a/spark/src/main/scala/org/apache/comet/GenerateDocs.scala
b/spark/src/main/scala/org/apache/comet/GenerateDocs.scala
index d96274368..d69eecfa0 100644
--- a/spark/src/main/scala/org/apache/comet/GenerateDocs.scala
+++ b/spark/src/main/scala/org/apache/comet/GenerateDocs.scala
@@ -50,7 +50,11 @@ object GenerateDocs {
w.write("| Config | Description | Default Value |\n".getBytes)
w.write("|--------|-------------|---------------|\n".getBytes)
for (conf <- confs) {
- w.write(s"| ${conf.key} | ${conf.doc.trim} |
${conf.defaultValueString} |\n".getBytes)
+ if (conf.defaultValue.isEmpty) {
+ w.write(s"| ${conf.key} | ${conf.doc.trim} | |\n".getBytes)
+ } else {
+ w.write(s"| ${conf.key} | ${conf.doc.trim} |
${conf.defaultValueString} |\n".getBytes)
+ }
}
} else {
w.write(s"${line.trim}\n".getBytes)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
