This is an automated email from the ASF dual-hosted git repository.
zhztheplayer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 7415d74c22 [VL] Make Velox memory manager capacity ratio configurable
(#12258)
7415d74c22 is described below
commit 7415d74c227cb3e930d933680f9a75d540372e7b
Author: Pratham Manja <[email protected]>
AuthorDate: Mon Jun 8 21:46:34 2026 +0530
[VL] Make Velox memory manager capacity ratio configurable (#12258)
---
cpp/core/config/GlutenConfig.h | 3 +++
cpp/velox/compute/VeloxBackend.cc | 16 +++++++++++++---
.../scala/org/apache/gluten/config/GlutenConfig.scala | 12 ++++++++++++
.../apache/gluten/config/GlutenRuntimeConfigSuite.scala | 15 +++++++++++++++
4 files changed, 43 insertions(+), 3 deletions(-)
diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h
index 2b8ba54595..16140a3ac3 100644
--- a/cpp/core/config/GlutenConfig.h
+++ b/cpp/core/config/GlutenConfig.h
@@ -42,6 +42,9 @@ const std::string kIgnoreMissingFiles =
"spark.sql.files.ignoreMissingFiles";
const std::string kSparkOverheadMemory =
"spark.gluten.memoryOverhead.size.in.bytes";
+const std::string kMemoryManagerCapacityRatio =
"spark.gluten.memory.manager.capacity.ratio";
+const double kMemoryManagerCapacityRatioDefault = 0.75;
+
const std::string kSparkOffHeapMemory =
"spark.gluten.memory.offHeap.size.in.bytes";
const std::string kSparkTaskOffHeapMemory =
"spark.gluten.memory.task.offHeap.size.in.bytes";
diff --git a/cpp/velox/compute/VeloxBackend.cc
b/cpp/velox/compute/VeloxBackend.cc
index 78b9f5ee04..816aba7c99 100644
--- a/cpp/velox/compute/VeloxBackend.cc
+++ b/cpp/velox/compute/VeloxBackend.cc
@@ -228,9 +228,19 @@ void VeloxBackend::init(
auto sparkOverhead = backendConf_->get<int64_t>(kSparkOverheadMemory);
int64_t memoryManagerCapacity;
if (sparkOverhead.has_value()) {
- // 0.75 * total overhead memory is used for Velox global memory manager.
- // FIXME: Make this configurable.
- memoryManagerCapacity = sparkOverhead.value() * 0.75;
+ // Get configurable ratio for Velox global memory manager capacity
+ auto capacityRatio =
backendConf_->get<double>(kMemoryManagerCapacityRatio);
+ double ratio = capacityRatio.has_value() ? capacityRatio.value() :
kMemoryManagerCapacityRatioDefault;
+
+ if (ratio <= 0.0 || ratio > 1.0) {
+ LOG(WARNING) << "Invalid memory manager capacity ratio: " << ratio
+ << ". Using default: " <<
kMemoryManagerCapacityRatioDefault;
+ ratio = kMemoryManagerCapacityRatioDefault;
+ }
+
+ memoryManagerCapacity = static_cast<int64_t>(sparkOverhead.value() *
ratio);
+ LOG(INFO) << "Using memory manager capacity ratio: " << ratio << "
(overhead: " << sparkOverhead.value()
+ << ", capacity: " << memoryManagerCapacity << ")";
} else {
memoryManagerCapacity = facebook::velox::memory::kMaxMemory;
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index 9c59097991..d198e6ffc4 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -1251,6 +1251,18 @@ object GlutenConfig extends ConfigRegistry {
.booleanConf
.createWithDefault(false)
+ val MEMORY_MANAGER_CAPACITY_RATIO =
+ buildConf("spark.gluten.memory.manager.capacity.ratio")
+ .internal()
+ .doc(
+ "Ratio of spark.gluten.memoryOverhead.size.in.bytes to allocate for
Velox global " +
+ "memory manager. The memory manager is used during spill
operations.")
+ .doubleConf
+ .checkValue(
+ ratio => ratio > 0.0 && ratio <= 1.0,
+ "Memory manager capacity ratio must be between 0.0 and 1.0")
+ .createWithDefault(0.75)
+
val TRANSFORM_PLAN_LOG_LEVEL =
buildConf("spark.gluten.sql.transform.logLevel")
.internal()
diff --git
a/gluten-ut/test/src/test/scala/org/apache/gluten/config/GlutenRuntimeConfigSuite.scala
b/gluten-ut/test/src/test/scala/org/apache/gluten/config/GlutenRuntimeConfigSuite.scala
index 87d73bde23..085b39973e 100644
---
a/gluten-ut/test/src/test/scala/org/apache/gluten/config/GlutenRuntimeConfigSuite.scala
+++
b/gluten-ut/test/src/test/scala/org/apache/gluten/config/GlutenRuntimeConfigSuite.scala
@@ -47,4 +47,19 @@ class GlutenRuntimeConfigSuite extends GlutenQueryTest with
SharedSparkSession {
conf.set(key, original)
}
}
+
+ test("Memory manager capacity ratio config validation") {
+
+ assert(GlutenConfig.MEMORY_MANAGER_CAPACITY_RATIO.defaultValue.get == 0.75)
+
+ withSQLConf(GlutenConfig.MEMORY_MANAGER_CAPACITY_RATIO.key -> "0.8") {
+
assert(GlutenConfig.get.getConf(GlutenConfig.MEMORY_MANAGER_CAPACITY_RATIO) ==
0.8)
+ }
+
+ intercept[IllegalArgumentException] {
+ withSQLConf(GlutenConfig.MEMORY_MANAGER_CAPACITY_RATIO.key -> "1.5") {
+ GlutenConfig.get.getConf(GlutenConfig.MEMORY_MANAGER_CAPACITY_RATIO)
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
