This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3e6263e204 [VL] Re-enable background IO threads by default (#7845)
3e6263e204 is described below
commit 3e6263e204aee9b5f1d1afe06102cb97251c08ed
Author: Hongze Zhang <[email protected]>
AuthorDate: Fri Nov 8 12:44:17 2024 +0800
[VL] Re-enable background IO threads by default (#7845)
---
.github/workflows/velox_backend.yml | 42 ----------------------
.../org/apache/gluten/test/MockVeloxBackend.java | 1 +
.../scala/org/apache/gluten/GlutenConfig.scala | 18 +++++-----
3 files changed, 9 insertions(+), 52 deletions(-)
diff --git a/.github/workflows/velox_backend.yml
b/.github/workflows/velox_backend.yml
index 6e9d0eab7c..737895be57 100644
--- a/.github/workflows/velox_backend.yml
+++ b/.github/workflows/velox_backend.yml
@@ -249,48 +249,6 @@ jobs:
--local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
--extra-conf=spark.gluten.ras.enabled=true
- run-tpc-test-ubuntu-iothreads:
- needs: build-native-lib-centos-7
- strategy:
- fail-fast: false
- matrix:
- spark: [ "spark-3.5" ]
- runs-on: ubuntu-20.04
- steps:
- - uses: actions/checkout@v2
- - name: Download All Native Artifacts
- uses: actions/download-artifact@v3
- with:
- name: velox-native-lib-centos-7-${{github.sha}}
- path: ./cpp/build/releases/
- - name: Download All Arrow Jar Artifacts
- uses: actions/download-artifact@v3
- with:
- name: arrow-jars-centos-7-${{github.sha}}
- path: /home/runner/.m2/repository/org/apache/arrow/
- - name: Setup java and maven
- run: |
- sudo apt-get update
- sudo apt-get install -y openjdk-8-jdk maven
- - name: Set environment variables
- run: |
- echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV
- - name: Build for Spark ${{ matrix.spark }}
- run: |
- cd $GITHUB_WORKSPACE/
- $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox
-DskipTests
- cd $GITHUB_WORKSPACE/tools/gluten-it
- $MVN_CMD clean install -P${{ matrix.spark }}
- - name: Build and run TPC-H / TPC-DS
- run: |
- cd $GITHUB_WORKSPACE/tools/gluten-it
- GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- --extra-conf=spark.gluten.sql.columnar.backend.velox.IOThreads=16
- GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
- --local --preset=velox --benchmark-type=ds --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \
- --extra-conf=spark.gluten.sql.columnar.backend.velox.IOThreads=16
-
run-tpc-test-ubuntu-oom:
needs: build-native-lib-centos-7
strategy:
diff --git
a/backends-velox/src/test/java/org/apache/gluten/test/MockVeloxBackend.java
b/backends-velox/src/test/java/org/apache/gluten/test/MockVeloxBackend.java
index b5ce30e39b..254e8e2830 100644
--- a/backends-velox/src/test/java/org/apache/gluten/test/MockVeloxBackend.java
+++ b/backends-velox/src/test/java/org/apache/gluten/test/MockVeloxBackend.java
@@ -71,6 +71,7 @@ public final class MockVeloxBackend {
private static SparkConf newSparkConf() {
final SparkConf conf = new SparkConf();
conf.set(GlutenConfig.SPARK_OFFHEAP_SIZE_KEY(), "1g");
+ conf.set(GlutenConfig.COLUMNAR_VELOX_CONNECTOR_IO_THREADS(), "0");
return conf;
}
}
diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
index a28a7d26b3..3e491eb275 100644
--- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -291,7 +291,7 @@ class GlutenConfig(conf: SQLConf) extends Logging {
def veloxSsdODirectEnabled: Boolean =
conf.getConf(COLUMNAR_VELOX_SSD_ODIRECT_ENABLED)
def veloxConnectorIOThreads: Int = {
- conf.getConf(COLUMNAR_VELOX_CONNECTOR_IO_THREADS)
+
conf.getConf(COLUMNAR_VELOX_CONNECTOR_IO_THREADS).getOrElse(numTaskSlotsPerExecutor)
}
def veloxSplitPreloadPerDriver: Integer =
conf.getConf(COLUMNAR_VELOX_SPLIT_PRELOAD_PER_DRIVER)
@@ -759,7 +759,9 @@ object GlutenConfig {
(AWS_S3_RETRY_MODE.key, AWS_S3_RETRY_MODE.defaultValueString),
(
COLUMNAR_VELOX_CONNECTOR_IO_THREADS.key,
- COLUMNAR_VELOX_CONNECTOR_IO_THREADS.defaultValueString),
+ conf.getOrElse(
+ NUM_TASK_SLOTS_PER_EXECUTOR.key,
+ NUM_TASK_SLOTS_PER_EXECUTOR.defaultValueString)),
(COLUMNAR_SHUFFLE_CODEC.key, ""),
(COLUMNAR_SHUFFLE_CODEC_BACKEND.key, ""),
("spark.hadoop.input.connect.timeout", "180000"),
@@ -1449,19 +1451,15 @@ object GlutenConfig {
.booleanConf
.createWithDefault(false)
- // FIXME: May cause issues when toggled on. Examples:
- // https://github.com/apache/incubator-gluten/issues/7161
- // https://github.com/facebookincubator/velox/issues/10173
val COLUMNAR_VELOX_CONNECTOR_IO_THREADS =
buildStaticConf("spark.gluten.sql.columnar.backend.velox.IOThreads")
.internal()
.doc(
- "Experimental: The Size of the IO thread pool in the Connector." +
- " This thread pool is used for split preloading and
DirectBufferedInput." +
- " The option is experimental. Toggling on it (setting a non-zero
value) may cause some" +
- " unexpected issues when application reaches some certain
conditions.")
+ "The Size of the IO thread pool in the Connector. " +
+ "This thread pool is used for split preloading and
DirectBufferedInput. " +
+ "By default, the value is the same as the maximum task slots per
Spark executor.")
.intConf
- .createWithDefault(0)
+ .createOptional
val COLUMNAR_VELOX_ASYNC_TIMEOUT =
buildStaticConf("spark.gluten.sql.columnar.backend.velox.asyncTimeoutOnTaskStopping")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
