(incubator-gluten) branch main updated: [GLUTEN-5884][VL] change default load quantum to 8M for local SSD cache (#8880)

[felixybw]

This is an automated email from the ASF dual-hosted git repository.

felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new c64ff69d24 [GLUTEN-5884][VL] change default load quantum to 8M for 
local SSD cache (#8880)
c64ff69d24 is described below

commit c64ff69d24a5da9186f573e64d9b3254d65d2aa0
Author: Yuan <yuan.z...@intel.com>
AuthorDate: Tue Mar 4 18:14:16 2025 +0000

    [GLUTEN-5884][VL] change default load quantum to 8M for local SSD cache 
(#8880)
    
    Velox is using 23 bits fro cache entry size
    facebookincubator/velox#10242
---
 .../src/main/scala/org/apache/gluten/config/VeloxConfig.scala         | 4 ++--
 cpp/velox/compute/VeloxBackend.cc                                     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git 
a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala 
b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
index 9fd8064d6e..e1f1c2880e 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
@@ -406,10 +406,10 @@ object VeloxConfig {
   val LOAD_QUANTUM =
     buildStaticConf("spark.gluten.sql.columnar.backend.velox.loadQuantum")
       .internal()
-      .doc("Set the load quantum for velox file scan, recommend to use the 
default value (256MB) " +
+      .doc("Set the load quantum for velox file scan, recommend to use the 
default value (8MB) " +
         "for performance consideration. If Velox cache is enabled, it can be 
8MB at most.")
       .bytesConf(ByteUnit.BYTE)
-      .createWithDefaultString("256MB")
+      .createWithDefaultString("8MB")
 
   val MAX_COALESCED_DISTANCE_BYTES =
     
buildStaticConf("spark.gluten.sql.columnar.backend.velox.maxCoalescedDistance")
diff --git a/cpp/velox/compute/VeloxBackend.cc 
b/cpp/velox/compute/VeloxBackend.cc
index c516a8460e..cd62da5c66 100644
--- a/cpp/velox/compute/VeloxBackend.cc
+++ b/cpp/velox/compute/VeloxBackend.cc
@@ -265,7 +265,7 @@ void VeloxBackend::initConnector() {
   connectorConfMap[velox::connector::hive::HiveConfig::kPrefetchRowGroups] =
       backendConf_->get<std::string>(kPrefetchRowGroups, "1");
   connectorConfMap[velox::connector::hive::HiveConfig::kLoadQuantum] =
-      backendConf_->get<std::string>(kLoadQuantum, "268435456"); // 256M
+      backendConf_->get<std::string>(kLoadQuantum, "8388608"); // 8M
   connectorConfMap[velox::connector::hive::HiveConfig::kFooterEstimatedSize] =
       backendConf_->get<std::string>(kDirectorySizeGuess, "32768"); // 32K
   connectorConfMap[velox::connector::hive::HiveConfig::kFilePreloadThreshold] =


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@gluten.apache.org
For additional commands, e-mail: commits-h...@gluten.apache.org