This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fd3af489a4 [memory](chunkallocator) disable chunkallocator when
reserved bytes == 0 (#14494)
fd3af489a4 is described below
commit fd3af489a418afad25017dde3e817579df839cca
Author: yiguolei <[email protected]>
AuthorDate: Wed Nov 23 17:12:53 2022 +0800
[memory](chunkallocator) disable chunkallocator when reserved bytes == 0
(#14494)
disable chunkallocator when reserved bytes == 0
disable chunkallocator by default
---
be/src/common/config.h | 2 +-
be/src/runtime/exec_env_init.cpp | 6 ------
be/src/runtime/mem_pool.cpp | 2 +-
be/src/runtime/memory/chunk_allocator.cpp | 18 ++++++++++--------
be/src/runtime/memory/chunk_allocator.h | 7 -------
be/test/runtime/memory/chunk_allocator_test.cpp | 2 +-
6 files changed, 13 insertions(+), 24 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index a9c9357905..554735b7c5 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -414,7 +414,7 @@ CONF_Bool(disable_mem_pools, "false");
// must larger than 0. and if larger than physical memory size, it will be set
to physical memory size.
// increase this variable can improve performance,
// but will acquire more free memory which can not be used by other modules.
-CONF_mString(chunk_reserved_bytes_limit, "10%");
+CONF_mString(chunk_reserved_bytes_limit, "0");
// 1024, The minimum chunk allocator size (in bytes)
CONF_Int32(min_chunk_reserved_bytes, "1024");
// Disable Chunk Allocator in Vectorized Allocator, this will reduce memory
cache.
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index e4cd3c7bab..735d8091ed 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -274,12 +274,6 @@ Status ExecEnv::_init_mem_env() {
int64_t chunk_reserved_bytes_limit =
ParseUtil::parse_mem_spec(config::chunk_reserved_bytes_limit,
MemInfo::mem_limit(),
MemInfo::physical_mem(), &is_percent);
- if (chunk_reserved_bytes_limit <= 0) {
- ss << "Invalid config chunk_reserved_bytes_limit value, must be a
percentage or "
- "positive bytes value or percentage: "
- << config::chunk_reserved_bytes_limit;
- return Status::InternalError(ss.str());
- }
chunk_reserved_bytes_limit =
BitUtil::RoundDown(chunk_reserved_bytes_limit,
config::min_chunk_reserved_bytes);
ChunkAllocator::init_instance(chunk_reserved_bytes_limit);
diff --git a/be/src/runtime/mem_pool.cpp b/be/src/runtime/mem_pool.cpp
index 1b56b50760..4c136cc019 100644
--- a/be/src/runtime/mem_pool.cpp
+++ b/be/src/runtime/mem_pool.cpp
@@ -143,7 +143,7 @@ Status MemPool::find_chunk(size_t min_size, bool
check_limits, bool free_old_chu
free_all();
}
Chunk chunk;
- RETURN_IF_ERROR(ChunkAllocator::instance()->allocate(chunk_size, &chunk));
+ RETURN_IF_ERROR(ChunkAllocator::instance()->allocate_align(chunk_size,
&chunk));
if (_mem_tracker) _mem_tracker->consume(chunk_size);
ASAN_POISON_MEMORY_REGION(chunk.data, chunk_size);
// Put it before the first free chunk. If no free chunks, it goes at the
end.
diff --git a/be/src/runtime/memory/chunk_allocator.cpp
b/be/src/runtime/memory/chunk_allocator.cpp
index b812734b4c..6ac8021648 100644
--- a/be/src/runtime/memory/chunk_allocator.cpp
+++ b/be/src/runtime/memory/chunk_allocator.cpp
@@ -152,14 +152,20 @@ ChunkAllocator::ChunkAllocator(size_t reserve_limit)
INT_GAUGE_METRIC_REGISTER(_chunk_allocator_metric_entity,
chunk_pool_reserved_bytes);
}
-Status ChunkAllocator::allocate(size_t size, Chunk* chunk) {
- CHECK((size > 0 && (size & (size - 1)) == 0));
-
+Status ChunkAllocator::allocate_align(size_t size, Chunk* chunk) {
+ CHECK(size > 0);
+ size = BitUtil::RoundUpToPowerOfTwo(size);
// fast path: allocate from current core arena
int core_id = CpuInfo::get_current_core();
chunk->size = size;
chunk->core_id = core_id;
+ if (_reserve_bytes_limit < 1) {
+ // allocate from system allocator
+ chunk->data = SystemAllocator::allocate(size);
+ return Status::OK();
+ }
+
if (_arenas[core_id]->pop_free_chunk(size, &chunk->data)) {
DCHECK_GE(_reserved_bytes, 0);
_reserved_bytes.fetch_sub(size);
@@ -205,7 +211,7 @@ Status ChunkAllocator::allocate(size_t size, Chunk* chunk) {
void ChunkAllocator::free(const Chunk& chunk) {
DCHECK(chunk.core_id != -1);
CHECK((chunk.size & (chunk.size - 1)) == 0);
- if (config::disable_mem_pools) {
+ if (config::disable_mem_pools || _reserve_bytes_limit < 1) {
SystemAllocator::free(chunk.data, chunk.size);
return;
}
@@ -242,10 +248,6 @@ void ChunkAllocator::free(const Chunk& chunk) {
_arenas[chunk.core_id]->push_free_chunk(chunk.data, chunk.size);
}
-Status ChunkAllocator::allocate_align(size_t size, Chunk* chunk) {
- return allocate(BitUtil::RoundUpToPowerOfTwo(size), chunk);
-}
-
void ChunkAllocator::free(uint8_t* data, size_t size) {
Chunk chunk;
chunk.data = data;
diff --git a/be/src/runtime/memory/chunk_allocator.h
b/be/src/runtime/memory/chunk_allocator.h
index 0d57e2bd30..de9ff70487 100644
--- a/be/src/runtime/memory/chunk_allocator.h
+++ b/be/src/runtime/memory/chunk_allocator.h
@@ -73,15 +73,8 @@ public:
void free(uint8_t* data, size_t size);
private:
- friend class MemPool;
-
ChunkAllocator(size_t reserve_limit);
- // Allocate a Chunk with a power-of-two length "size".
- // Return true if success and allocated chunk is saved in "chunk".
- // Otherwise return false.
- Status allocate(size_t size, Chunk* Chunk);
-
private:
static ChunkAllocator* _s_instance;
diff --git a/be/test/runtime/memory/chunk_allocator_test.cpp
b/be/test/runtime/memory/chunk_allocator_test.cpp
index b3854724eb..47fce96ce8 100644
--- a/be/test/runtime/memory/chunk_allocator_test.cpp
+++ b/be/test/runtime/memory/chunk_allocator_test.cpp
@@ -30,7 +30,7 @@ namespace doris {
TEST(ChunkAllocatorTest, Normal) {
for (size_t size = 4096; size <= 1024 * 1024; size <<= 1) {
Chunk chunk;
- EXPECT_TRUE(ChunkAllocator::instance()->allocate(size, &chunk).ok());
+ EXPECT_TRUE(ChunkAllocator::instance()->allocate_align(size,
&chunk).ok());
EXPECT_NE(nullptr, chunk.data);
EXPECT_EQ(size, chunk.size);
ChunkAllocator::instance()->free(chunk);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
