This is an automated email from the ASF dual-hosted git repository. marin-ma pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 065c073de5 [VL] Fix gen config script (#12419)
065c073de5 is described below
commit 065c073de591ef6cb3c16b7af2ca37a6e6b639d7
Author: Rong Ma <[email protected]>
AuthorDate: Thu Jul 2 13:41:17 2026 +0100
[VL] Fix gen config script (#12419)
---
dev/gen-all-config-docs.sh | 2 +-
docs/Configuration.md | 32 +++++++++++-----------
docs/velox-configuration.md | 6 ++--
.../gluten/config/AllGlutenConfiguration.scala | 25 ++++++-----------
pom.xml | 16 +++++++++++
5 files changed, 44 insertions(+), 37 deletions(-)
diff --git a/dev/gen-all-config-docs.sh b/dev/gen-all-config-docs.sh
index 01ae989a26..d58f7b3a43 100755
--- a/dev/gen-all-config-docs.sh
+++ b/dev/gen-all-config-docs.sh
@@ -22,7 +22,7 @@ MVN_CMD="${BASEDIR}/../build/mvn"
GLUTEN_UPDATE="${GLUTEN_UPDATE:-1}" \
${MVN_CMD} clean test \
- -Pbackends-velox -pl backends-velox -am \
+ -Pspark-4.0 -Pscala-2.13 -Pbackends-velox -pl backends-velox -am \
-Dtest=none \
-DfailIfNoTests=false \
-DwildcardSuites=org.apache.gluten.config.AllGlutenConfiguration,org.apache.gluten.config.AllVeloxConfiguration
diff --git a/docs/Configuration.md b/docs/Configuration.md
index efc3e94291..0e4bc90b0c 100644
--- a/docs/Configuration.md
+++ b/docs/Configuration.md
@@ -34,22 +34,22 @@ nav_order: 15
| spark.gluten.memory.reservationBlockSize | 🔄
Dynamic | 8MB | Block size of native reservation listener
reserve memory from Spark.
[...]
| spark.gluten.numTaskSlotsPerExecutor | 🔄
Dynamic | -1 | Must provide default value since non-execution
operations (e.g. org.apache.spark.sql.Dataset#summary) doesn't propagate
configurations using
org.apache.spark.sql.execution.SQLExecution#withSQLConfPropagated
[...]
| spark.gluten.shuffleWriter.bufferSize | 🔄
Dynamic | <undefined> |
-| spark.gluten.soft-affinity.duplicateReading.maxCacheItems | 🔄
Dynamic | 10000 | Enable Soft Affinity duplicate reading
detection
[...]
-| spark.gluten.soft-affinity.duplicateReadingDetect.enabled | 🔄
Dynamic | false | If true, Enable Soft Affinity duplicate
reading detection
[...]
-| spark.gluten.soft-affinity.enabled | 🔄
Dynamic | false | Whether to enable Soft Affinity scheduling.
[...]
-| spark.gluten.soft-affinity.min.target-hosts | 🔄
Dynamic | 1 | For on HDFS, if there are already target
hosts, and then prefer to use the original target hosts to schedule
[...]
-| spark.gluten.soft-affinity.replications.num | 🔄
Dynamic | 2 | Calculate the number of the replications for
scheduling to the target executors per file
[...]
-| spark.gluten.sql.adaptive.costEvaluator.enabled | âš“
Static | true | If true, use
org.apache.spark.sql.execution.adaptive.GlutenCostEvaluator as custom cost
evaluator class, else follow the configuration
spark.sql.adaptive.customCostEvaluatorClass.
[...]
-| spark.gluten.sql.ansiFallback.enabled | 🔄
Dynamic | true | When true (default), Gluten will fall back to
Spark when ANSI mode is enabled. When false, Gluten will attempt to execute in
ANSI mode.
[...]
-| spark.gluten.sql.cacheWholeStageTransformerContext | 🔄
Dynamic | false | When true, `WholeStageTransformer` will cache
the `WholeStageTransformerContext` when executing. It is used to get substrait
plan node and native plan string.
[...]
-| spark.gluten.sql.collapseGetJsonObject.enabled | 🔄
Dynamic | false | Collapse nested get_json_object functions as
one for optimization.
[...]
-| spark.gluten.sql.columnar.appendData | 🔄
Dynamic | true | Enable or disable columnar v2 command append
data.
[...]
-| spark.gluten.sql.columnar.arrowUdf | 🔄
Dynamic | true | Enable or disable columnar arrow udf.
[...]
-| spark.gluten.sql.columnar.batchscan | 🔄
Dynamic | true | Enable or disable columnar batchscan.
[...]
-| spark.gluten.sql.columnar.broadcastExchange | 🔄
Dynamic | true | Enable or disable columnar broadcastExchange.
[...]
-| spark.gluten.sql.columnar.broadcastJoin | 🔄
Dynamic | true | Enable or disable columnar broadcastJoin.
[...]
-| spark.gluten.sql.columnar.broadcastNestedLoopJoin.enabled | 🔄
Dynamic | true | Enable or disable columnar
broadcastNestedLoopJoin.
[...]
-| spark.gluten.sql.columnar.cartesianProduct.enabled | 🔄
Dynamic | true | Enable or disable columnar cartesianProduct.
[...]
+| spark.gluten.soft-affinity.duplicateReading.maxCacheItems | 🔄
Dynamic | 10000 | Enable Soft Affinity duplicate reading
detection
[...]
+| spark.gluten.soft-affinity.duplicateReadingDetect.enabled | 🔄
Dynamic | false | If true, Enable Soft Affinity duplicate
reading detection
[...]
+| spark.gluten.soft-affinity.enabled | 🔄
Dynamic | false | Whether to enable Soft Affinity scheduling.
[...]
+| spark.gluten.soft-affinity.min.target-hosts | 🔄
Dynamic | 1 | For on HDFS, if there are already target
hosts, and then prefer to use the original target hosts to schedule
[...]
+| spark.gluten.soft-affinity.replications.num | 🔄
Dynamic | 2 | Calculate the number of the replications for
scheduling to the target executors per file
[...]
+| spark.gluten.sql.adaptive.costEvaluator.enabled | âš“
Static | true | If true, use
org.apache.spark.sql.execution.adaptive.GlutenCostEvaluator as custom cost
evaluator class, else follow the configuration
spark.sql.adaptive.customCostEvaluatorClass.
[...]
+| spark.gluten.sql.ansiFallback.enabled | 🔄
Dynamic | true | When true (default), Gluten will fall back to
Spark when ANSI mode is enabled. When false, Gluten will attempt to execute in
ANSI mode.
[...]
+| spark.gluten.sql.cacheWholeStageTransformerContext | 🔄
Dynamic | false | When true, `WholeStageTransformer` will cache
the `WholeStageTransformerContext` when executing. It is used to get substrait
plan node and native plan string.
[...]
+| spark.gluten.sql.collapseGetJsonObject.enabled | 🔄
Dynamic | false | Collapse nested get_json_object functions as
one for optimization.
[...]
+| spark.gluten.sql.columnar.appendData | 🔄
Dynamic | true | Enable or disable columnar v2 command append
data.
[...]
+| spark.gluten.sql.columnar.arrowUdf | 🔄
Dynamic | true | Enable or disable columnar arrow udf.
[...]
+| spark.gluten.sql.columnar.batchscan | 🔄
Dynamic | true | Enable or disable columnar batchscan.
[...]
+| spark.gluten.sql.columnar.broadcastExchange | 🔄
Dynamic | true | Enable or disable columnar broadcastExchange.
[...]
+| spark.gluten.sql.columnar.broadcastJoin | 🔄
Dynamic | true | Enable or disable columnar broadcastJoin.
[...]
+| spark.gluten.sql.columnar.broadcastNestedLoopJoin.enabled | 🔄
Dynamic | true | Enable or disable columnar
broadcastNestedLoopJoin.
[...]
+| spark.gluten.sql.columnar.cartesianProduct.enabled | 🔄
Dynamic | true | Enable or disable columnar cartesianProduct.
[...]
| spark.gluten.sql.columnar.cast.avg | 🔄
Dynamic | true |
| spark.gluten.sql.columnar.coalesce | 🔄
Dynamic | true | Enable or disable columnar coalesce.
[...]
| spark.gluten.sql.columnar.collectLimit | 🔄
Dynamic | true | Enable or disable columnar collectLimit.
[...]
diff --git a/docs/velox-configuration.md b/docs/velox-configuration.md
index b0e1afdaa1..9a50e5ec8a 100644
--- a/docs/velox-configuration.md
+++ b/docs/velox-configuration.md
@@ -27,7 +27,7 @@ nav_order: 16
| spark.gluten.sql.columnar.backend.velox.cudf.memoryResource
| âš“ Static | async | GPU RMM memory resource.
[...]
| spark.gluten.sql.columnar.backend.velox.cudf.shuffleMaxPrefetchBytes
| 🔄 Dynamic | 1028MB | Maximum bytes to prefetch in CPU
memory during GPU shuffle read while waiting for GPU available.
[...]
| spark.gluten.sql.columnar.backend.velox.directorySizeGuess
| âš“ Static | 32KB | Deprecated, rename to
spark.gluten.sql.columnar.backend.velox.footerEstimatedSize
[...]
-| spark.gluten.sql.columnar.backend.velox.enableTimestampNtzValidation
| 🔄 Dynamic | true | Enable validation fallback for
TimestampNTZ type. When true (default), any plan containing TimestampNTZ will
fall back to Spark execution. Set to false during development/testing of
TimestampNTZ support to allow native execution.
[...]
+| spark.gluten.sql.columnar.backend.velox.enableTimestampNtzValidation
| 🔄 Dynamic | false | Enable validation fallback for
TimestampNTZ type. When true, any plan containing TimestampNTZ will fall back
to Spark execution. When false, allows native execution for TimestampNTZ scan.
[...]
| spark.gluten.sql.columnar.backend.velox.fileHandleCacheEnabled
| âš“ Static | false | Disables caching if false. File
handle cache should be disabled if files are mutable, i.e. file content may
change while file path stays the same.
[...]
| spark.gluten.sql.columnar.backend.velox.filePreloadThreshold
| âš“ Static | 1MB | Set the file preload threshold for
velox file scan, refer to Velox's file-preload-threshold
[...]
| spark.gluten.sql.columnar.backend.velox.floatingPointMode
| 🔄 Dynamic | loose | Config used to control the tolerance
of floating point operations alignment with Spark. When the mode is set to
strict, flushing is disabled for sum(float/double)and avg(float/double). When
set to loose, flushing will be enabled.
[...]
@@ -49,7 +49,6 @@ nav_order: 16
| spark.gluten.sql.columnar.backend.velox.maxSpillFileSize
| 🔄 Dynamic | 1GB | The maximum size of a single spill
file created
[...]
| spark.gluten.sql.columnar.backend.velox.maxSpillLevel
| 🔄 Dynamic | 4 | The max allowed spilling level with
zero being the initial spilling level
[...]
| spark.gluten.sql.columnar.backend.velox.maxSpillRunRows
| 🔄 Dynamic | 3M | The maximum row size of a single spill
run
[...]
-| spark.gluten.sql.columnar.backend.velox.parquetMaxTargetFileSize
| 🔄 Dynamic | 0b | The target file size for each
output file when writing parquet data. 0 means no limit on target file size,
and the actual file size will be determined by other factors such as max
partition number and shuffle batch size.
[...]
| spark.gluten.sql.columnar.backend.velox.memCacheSize
| âš“ Static | 1GB | The memory cache size
[...]
| spark.gluten.sql.columnar.backend.velox.memInitCapacity
| 🔄 Dynamic | 8MB | The initial memory capacity to reserve
for a newly created Velox query memory pool.
[...]
|
spark.gluten.sql.columnar.backend.velox.memoryPoolCapacityTransferAcrossTasks
| 🔄 Dynamic | true | Whether to allow memory capacity transfer
between memory pools from different tasks.
[...]
@@ -57,6 +56,7 @@ nav_order: 16
| spark.gluten.sql.columnar.backend.velox.orc.scan.enabled
| 🔄 Dynamic | true | Enable velox orc scan. If disabled,
vanilla spark orc scan will be used.
[...]
| spark.gluten.sql.columnar.backend.velox.orcUseColumnNames
| 🔄 Dynamic | true | Maps table field names to file field
names using names, not indices for ORC files.
[...]
| spark.gluten.sql.columnar.backend.velox.parquet.pageSizeBytes
| 🔄 Dynamic | 1MB | The page size in bytes is for
compression.
[...]
+| spark.gluten.sql.columnar.backend.velox.parquetMaxTargetFileSize
| 🔄 Dynamic | 0b | The target file size for each output
file when writing data. 0 means no limit on target file size, and the actual
file size will be determined by other factors such as max partition number and
shuffle batch size.
[...]
| spark.gluten.sql.columnar.backend.velox.parquetUseColumnNames
| 🔄 Dynamic | true | Maps table field names to file field
names using names, not indices for Parquet files.
[...]
| spark.gluten.sql.columnar.backend.velox.prefetchRowGroups
| âš“ Static | 1 | Set the prefetch row groups for velox
file scan
[...]
| spark.gluten.sql.columnar.backend.velox.queryTraceEnabled
| 🔄 Dynamic | false | Enable query tracing flag.
[...]
@@ -82,7 +82,7 @@ nav_order: 16
| spark.gluten.sql.enable.enhancedFeatures
| 🔄 Dynamic | true | Enable some features including iceberg
native write and other features.
[...]
| spark.gluten.sql.rewrite.castArrayToString
| 🔄 Dynamic | true | When true, rewrite `cast(array as
String)` to `concat('[', array_join(array, ', ', null), ']')` to allow
offloading to Velox.
[...]
| spark.gluten.velox.broadcast.build.targetBytesPerThread
| âš“ Static | 32MB | It is used to calculate the number of
hash table build threads. Based on our testing across various thresholds (1MB
to 128MB), we recommend a value of 32MB or 64MB, as these consistently provided
the most significant performance gains.
[...]
-| spark.gluten.velox.broadcastBuild.mergeBatches
| 🔄 Dynamic | false | If enabled, all columnar batches in a
broadcast build relation will be serialized into a single buffer to reduce the
number of addInput calls in HashBuild operator. This can significantly improve
BHJ performance when the broadcast table has many small batches, but may
increase driver-side peak memory and is not suitable for very large broadcasts.
|
+| spark.gluten.velox.broadcastBuild.mergeBatches
| 🔄 Dynamic | false | If enabled, all columnar batches in a
broadcast build relation will be serialized into a single buffer to reduce the
number of addInput calls in HashBuild operator. This can significantly improve
BHJ performance when the broadcast table has many small batches, but may
increase driver-side peak memory and is not suitable for very large broadcasts.
[...]
| spark.gluten.velox.castFromVarcharAddTrimNode
| 🔄 Dynamic | false | If true, will add a trim node which
has the same semantic as vanilla Spark to CAST-from-varchar.Otherwise, do
nothing.
[...]
## Gluten Velox backend *experimental* configurations
diff --git
a/gluten-substrait/src/test/scala/org/apache/gluten/config/AllGlutenConfiguration.scala
b/gluten-substrait/src/test/scala/org/apache/gluten/config/AllGlutenConfiguration.scala
index 37545b39ae..d51769d72a 100644
---
a/gluten-substrait/src/test/scala/org/apache/gluten/config/AllGlutenConfiguration.scala
+++
b/gluten-substrait/src/test/scala/org/apache/gluten/config/AllGlutenConfiguration.scala
@@ -27,7 +27,6 @@ import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path, Paths, StandardOpenOption}
import scala.collection.JavaConverters._
-import scala.collection.Traversable
import scala.io.Source
/**
@@ -210,7 +209,7 @@ object AllGlutenConfiguration {
StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING)
} else {
- assertFileContent(path, lines, regenScript)
+ assertFileContent(path, lines.toSeq, regenScript)
}
}
@@ -223,35 +222,27 @@ object AllGlutenConfiguration {
* source file path
* @param regenScript
* regeneration script
- * @param splitFirstExpectedLine
- * whether to split the first expected line into multiple lines by EOL
*/
def assertFileContent(
path: Path,
- expectedLines: Traversable[String],
- regenScript: String,
- splitFirstExpectedLine: Boolean = false)(implicit
+ expectedLines: Seq[String],
+ regenScript: String)(implicit
prettifier: Prettifier,
pos: Position): Unit = {
val fileSource = Source.fromFile(path.toUri, StandardCharsets.UTF_8.name())
try {
- def expectedLinesIter = if (splitFirstExpectedLine) {
- Source.fromString(expectedLines.head).getLines()
- } else {
- expectedLines.toIterator
- }
- val fileLinesIter = fileSource.getLines()
+ val fileLines = fileSource.getLines().toSeq
val regenerationHint = s"The file ($path) is out of date. " + {
if (regenScript != null && regenScript.nonEmpty) {
s" Please regenerate it by running `${regenScript.stripMargin}`. "
} else ""
}
- val fileLineCount = fileLinesIter.length
+ val fileLineCount = fileLines.length
withClue(s"Line number is not expected. $regenerationHint") {
- assertResult(expectedLinesIter.size)(fileLineCount)(prettifier, pos)
+ assertResult(expectedLines.size)(fileLineCount)(prettifier, pos)
}
- fileLinesIter.zipWithIndex
- .zip(expectedLinesIter)
+ fileLines.zipWithIndex
+ .zip(expectedLines)
.foreach {
case ((lineInFile, lineIndex), expectedLine) =>
val lineNum = lineIndex + 1
diff --git a/pom.xml b/pom.xml
index d69c773f64..f74955d061 100644
--- a/pom.xml
+++ b/pom.xml
@@ -945,6 +945,22 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
+ <execution>
+ <id>enforce-spark-profile</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <requireActiveProfile>
+ <!-- Spark 4.0 requires Java 17+ -->
+
<profiles>spark-3.3,spark-3.4,spark-3.5,spark-4.0,spark-4.1</profiles>
+ <all>false</all>
+ <message>Missing spark version profile:
-Pspark-<version></message>
+ </requireActiveProfile>
+ </rules>
+ </configuration>
+ </execution>
<execution>
<id>enforce-versions</id>
<goals>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
