This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 27a74d8c00 [VL] Use MallocAllocator for AsyncDataCache when the system
page size is not 4 KB (#13605)
27a74d8c00 is described below
commit 27a74d8c006ca77ec4fa22b9d88cefba999a2951
Author: WangGuangxin <[email protected]>
AuthorDate: Thu Nov 12 01:43:38 2026 +0800
[VL] Use MallocAllocator for AsyncDataCache when the system page size is
not 4 KB (#13605)
---
cpp/velox/compute/VeloxBackend.cc | 12 ++++++++++--
cpp/velox/compute/VeloxBackend.h | 3 ++-
2 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/cpp/velox/compute/VeloxBackend.cc
b/cpp/velox/compute/VeloxBackend.cc
index 449dfb7382..6ad6459892 100644
--- a/cpp/velox/compute/VeloxBackend.cc
+++ b/cpp/velox/compute/VeloxBackend.cc
@@ -397,9 +397,17 @@ void VeloxBackend::initCache() {
uint64_t memCacheSize = backendConf_->get<uint64_t>(kVeloxMemCacheSize,
kVeloxMemCacheSizeDefault);
uint64_t ssdCacheSize = backendConf_->get<uint64_t>(kVeloxSsdCacheSize,
kVeloxSsdCacheSizeDefault);
- velox::memory::MmapAllocator::Options options;
+ velox::memory::MemoryAllocator::Options options;
options.capacity = memCacheSize;
- cacheAllocator_ = std::make_shared<velox::memory::MmapAllocator>(options);
+ // MmapAllocator tracks memory in 4 KB pages and refuses to start when the
system page size differs (e.g. 64 KB
+ // pages on aarch64 kernels); fall back to MallocAllocator there.
+ if (velox::memory::MmapAllocator::isPageSizeSupported()) {
+ cacheAllocator_ =
std::make_shared<velox::memory::MmapAllocator>(options);
+ } else {
+ LOG(WARNING) << "System page size " <<
velox::memory::MmapAllocator::systemPageSize()
+ << " is not supported by MmapAllocator; using
MallocAllocator for AsyncDataCache";
+ cacheAllocator_ =
std::make_shared<velox::memory::MallocAllocator>(options);
+ }
if (ssdCacheSize == 0) {
LOG(INFO) << "AsyncDataCache will do memory caching only as ssd cache
size is 0";
// TODO: this is not tracked by Spark.
diff --git a/cpp/velox/compute/VeloxBackend.h b/cpp/velox/compute/VeloxBackend.h
index e6ac202ac1..0bd38a0835 100644
--- a/cpp/velox/compute/VeloxBackend.h
+++ b/cpp/velox/compute/VeloxBackend.h
@@ -27,6 +27,7 @@
#include "velox/common/caching/AsyncDataCache.h"
#include "velox/common/config/Config.h"
+#include "velox/common/memory/MallocAllocator.h"
#include "velox/common/memory/MmapAllocator.h"
#include "velox/connectors/Connector.h"
@@ -138,7 +139,7 @@ class VeloxBackend {
std::unique_ptr<folly::Executor> spillExecutor_;
std::unique_ptr<folly::Executor> ioExecutor_;
std::unique_ptr<folly::Executor> ssdCacheExecutor_;
- std::shared_ptr<facebook::velox::memory::MmapAllocator> cacheAllocator_;
+ std::shared_ptr<facebook::velox::memory::MemoryAllocator> cacheAllocator_;
std::shared_ptr<facebook::velox::config::ConfigBase> hiveConnectorConfig_;
std::string cachePathPrefix_;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]