This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4c182b8abc9 [enhance](HDFS) Add one config to control hdfs memory
limiter (#37547)
4c182b8abc9 is described below
commit 4c182b8abc9cc26c3595a37c35c6c05423820e5a
Author: AlexYue <[email protected]>
AuthorDate: Wed Jul 10 21:43:45 2024 +0800
[enhance](HDFS) Add one config to control hdfs memory limiter (#37547)
Previously when loading into hdfs cluster, we would try to do memory
control. Though it's one approximate memory recording and limit control
solution, it's not so accuracy. So we add one config to control whether
using the memory limit controller or not.
---
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 2 ++
be/src/io/fs/hdfs_file_writer.cpp | 6 ++++++
3 files changed, 10 insertions(+)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index ef00802cab2..d016201ddfc 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1322,6 +1322,8 @@ DEFINE_mBool(enable_parquet_page_index, "true");
DEFINE_mBool(ignore_not_found_file_in_external_table, "true");
+DEFINE_mBool(enable_hdfs_mem_limiter, "true");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 5c6082d0b13..eec7d585f5d 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1412,6 +1412,8 @@ DECLARE_mBool(enable_parquet_page_index);
// Default is true, if set to false, the not found file will result in query
failure.
DECLARE_mBool(ignore_not_found_file_in_external_table);
+DECLARE_mBool(enable_hdfs_mem_limiter);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/io/fs/hdfs_file_writer.cpp
b/be/src/io/fs/hdfs_file_writer.cpp
index ee9304a3637..ceff2cc429a 100644
--- a/be/src/io/fs/hdfs_file_writer.cpp
+++ b/be/src/io/fs/hdfs_file_writer.cpp
@@ -82,6 +82,9 @@ public:
#if defined(USE_LIBHDFS3) || defined(BE_TEST)
return Status::OK();
#else
+ if (!config::enable_hdfs_mem_limiter) {
+ return Status::OK();
+ }
auto unit = config::hdfs_jni_write_sleep_milliseconds;
std::default_random_engine rng = make_random_engine();
std::uniform_int_distribution<uint32_t> u(unit, 2 * unit);
@@ -106,6 +109,9 @@ public:
void release_memory(size_t memory_size) {
#if defined(USE_LIBHDFS3) || defined(BE_TEST)
#else
+ if (!config::enable_hdfs_mem_limiter) {
+ return;
+ }
std::unique_lock lck {cur_memory_latch};
size_t origin_size = cur_memory_comsuption;
cur_memory_comsuption -= memory_size;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
