This is an automated email from the ASF dual-hosted git repository. felixybw pushed a commit to branch revert-8880-wip_local_cache_refine in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
commit 70d8d5344b379d176f5adbc28461054730eab4f2 Author: BInwei Yang <[email protected]> AuthorDate: Tue Mar 4 17:54:02 2025 -0800 Revert "[GLUTEN-5884][VL] change default load quantum to 8M for local SSD cac…" This reverts commit c64ff69d24a5da9186f573e64d9b3254d65d2aa0. --- .../src/main/scala/org/apache/gluten/config/VeloxConfig.scala | 4 ++-- cpp/velox/compute/VeloxBackend.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala index e1f1c2880e..9fd8064d6e 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala @@ -406,10 +406,10 @@ object VeloxConfig { val LOAD_QUANTUM = buildStaticConf("spark.gluten.sql.columnar.backend.velox.loadQuantum") .internal() - .doc("Set the load quantum for velox file scan, recommend to use the default value (8MB) " + + .doc("Set the load quantum for velox file scan, recommend to use the default value (256MB) " + "for performance consideration. If Velox cache is enabled, it can be 8MB at most.") .bytesConf(ByteUnit.BYTE) - .createWithDefaultString("8MB") + .createWithDefaultString("256MB") val MAX_COALESCED_DISTANCE_BYTES = buildStaticConf("spark.gluten.sql.columnar.backend.velox.maxCoalescedDistance") diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc index cd62da5c66..c516a8460e 100644 --- a/cpp/velox/compute/VeloxBackend.cc +++ b/cpp/velox/compute/VeloxBackend.cc @@ -265,7 +265,7 @@ void VeloxBackend::initConnector() { connectorConfMap[velox::connector::hive::HiveConfig::kPrefetchRowGroups] = backendConf_->get<std::string>(kPrefetchRowGroups, "1"); connectorConfMap[velox::connector::hive::HiveConfig::kLoadQuantum] = - backendConf_->get<std::string>(kLoadQuantum, "8388608"); // 8M + backendConf_->get<std::string>(kLoadQuantum, "268435456"); // 256M connectorConfMap[velox::connector::hive::HiveConfig::kFooterEstimatedSize] = backendConf_->get<std::string>(kDirectorySizeGuess, "32768"); // 32K connectorConfMap[velox::connector::hive::HiveConfig::kFilePreloadThreshold] = --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
