This is an automated email from the ASF dual-hosted git repository.
marin-ma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new bd2c1ce6be [VL] Support cudf.allowCpuFallback config for Velox cuDF
backend (#12451)
bd2c1ce6be is described below
commit bd2c1ce6bee5a88a706c57e3c24dd12198f6bfb7
Author: Reema <[email protected]>
AuthorDate: Tue Jul 7 08:11:30 2026 +0000
[VL] Support cudf.allowCpuFallback config for Velox cuDF backend (#12451)
---
.../src/main/scala/org/apache/gluten/config/VeloxConfig.scala | 6 ++++++
cpp/core/config/GlutenConfig.h | 2 ++
cpp/velox/compute/VeloxBackend.cc | 4 +++-
docs/velox-configuration.md | 1 +
4 files changed, 12 insertions(+), 1 deletion(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
index 5864529058..81ae87a6fc 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
@@ -779,6 +779,12 @@ object VeloxConfig extends ConfigRegistry {
.booleanConf
.createWithDefault(true)
+ val CUDF_ALLOW_CPU_FALLBACK =
+
buildStaticConf("spark.gluten.sql.columnar.backend.velox.cudf.allowCpuFallback")
+ .doc("Allow cuDF to fall back to CPU execution for unsupported
operators.")
+ .booleanConf
+ .createWithDefault(true)
+
val CUDF_CONCURRENT_GPU_TASKS =
buildStaticConf("spark.gluten.sql.columnar.backend.velox.cudf.concurrentGpuTasks")
.doc("The number of concurrent GPU tasks to run.")
diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h
index 3dd99e4cf3..9c55628991 100644
--- a/cpp/core/config/GlutenConfig.h
+++ b/cpp/core/config/GlutenConfig.h
@@ -110,6 +110,8 @@ const std::string kCudfEnabled =
"spark.gluten.sql.columnar.cudf";
constexpr bool kCudfEnabledDefault = false;
const std::string kDebugCudf = "spark.gluten.sql.debug.cudf";
const std::string kDebugCudfDefault = "false";
+const std::string kCudfAllowCpuFallback =
"spark.gluten.sql.columnar.backend.velox.cudf.allowCpuFallback";
+const std::string kCudfAllowCpuFallbackDefault = "true";
std::unordered_map<std::string, std::string>
parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t
planDataLength);
diff --git a/cpp/velox/compute/VeloxBackend.cc
b/cpp/velox/compute/VeloxBackend.cc
index 4ba7dd7a73..d23ab1782c 100644
--- a/cpp/velox/compute/VeloxBackend.cc
+++ b/cpp/velox/compute/VeloxBackend.cc
@@ -202,7 +202,9 @@ void VeloxBackend::init(
{velox::cudf_velox::CudfConfig::kCudfMemoryResource,
backendConf_->get(kCudfMemoryResource, kCudfMemoryResourceDefault)},
{velox::cudf_velox::CudfConfig::kCudfMemoryPercent,
- backendConf_->get(kCudfMemoryPercent, kCudfMemoryPercentDefault)}};
+ backendConf_->get(kCudfMemoryPercent, kCudfMemoryPercentDefault)},
+ {velox::cudf_velox::CudfConfig::kCudfAllowCpuFallback,
+ backendConf_->get(kCudfAllowCpuFallback,
kCudfAllowCpuFallbackDefault)}};
auto& cudfConfig = velox::cudf_velox::CudfConfig::getInstance();
cudfConfig.initialize(std::move(options));
velox::cudf_velox::registerCudf();
diff --git a/docs/velox-configuration.md b/docs/velox-configuration.md
index 5f7c49afc4..44f3f5949a 100644
--- a/docs/velox-configuration.md
+++ b/docs/velox-configuration.md
@@ -19,6 +19,7 @@ nav_order: 16
| spark.gluten.sql.columnar.backend.velox.cacheEnabled
| ⚓ Static | false | Enable Velox cache, default off. It's
recommended to enablesoft-affinity as well when enable velox cache.
[...]
| spark.gluten.sql.columnar.backend.velox.cachePrefetchMinPct
| ⚓ Static | 0 | Set prefetch cache min pct for velox
file scan
[...]
| spark.gluten.sql.columnar.backend.velox.checkUsageLeak
| ⚓ Static | true | Enable check memory usage leak.
[...]
+| spark.gluten.sql.columnar.backend.velox.cudf.allowCpuFallback
| ⚓ Static | true | Allow cuDF to fall back to CPU
execution for unsupported operators.
[...]
| spark.gluten.sql.columnar.backend.velox.cudf.batchSize
| 🔄 Dynamic | 2147483647 | Cudf input batch size after shuffle
reader
[...]
| spark.gluten.sql.columnar.backend.velox.cudf.concurrentGpuTasks
| ⚓ Static | 1 | The number of concurrent GPU tasks to
run.
[...]
| spark.gluten.sql.columnar.backend.velox.cudf.enableTableScan
| ⚓ Static | false | Enable cudf table scan
[...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
