This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 8fa99196dc8 branch-4.1: [perf](load) Optimize write performance with
improved backpressure control (#67728)
8fa99196dc8 is described below
commit 8fa99196dc84f10e081359113d1f3529aafc85a7
Author: hui lai <[email protected]>
AuthorDate: Thu Sep 10 09:28:03 2026 +0800
branch-4.1: [perf](load) Optimize write performance with improved
backpressure control (#67728)
### What problem does this PR solve?
Backport #66847 to branch-4.1.
For S3-backed cloud loads, a memtable flush can finish after submitting
asynchronous uploads, so flush-running count does not represent
downstream upload pressure. This change detects S3 storage and uses the
S3 upload queue for backpressure, falling back to the existing
flush-running limit when the upload pool is unavailable. HDFS retains
flush-count backpressure.
### Branch-4.1 adaptation
branch-4.1 does not contain WriteRequestType::GROUP or
CloudGroupRowsetBuilder. The row-binlog GROUP-specific changes from the
original PR are therefore omitted. CloudRowsetBuilder is final in this
branch, so is_s3_storage is non-virtual. Local-storage flush limits
remain unchanged.
### Release note
Improve load backpressure for S3-backed cloud loads. Higher concurrency
can increase memory use.
### Check List (For Author)
- Test: Compilation and tests skipped as requested. clang-format 16 and
git diff --check passed.
- Behavior changed: Yes; S3-backed loads use upload queue pressure. HDFS
and local-storage limits remain unchanged.
- Does this need documentation: No.
---
be/src/cloud/cloud_delta_writer.cpp | 16 ++++++++++++++--
be/src/cloud/cloud_rowset_builder.cpp | 8 ++++++++
be/src/cloud/cloud_rowset_builder.h | 2 ++
3 files changed, 24 insertions(+), 2 deletions(-)
diff --git a/be/src/cloud/cloud_delta_writer.cpp
b/be/src/cloud/cloud_delta_writer.cpp
index 7a7fd00bb68..b85e83be20c 100644
--- a/be/src/cloud/cloud_delta_writer.cpp
+++ b/be/src/cloud/cloud_delta_writer.cpp
@@ -25,6 +25,8 @@
#include "load/memtable/memtable_memory_limiter.h"
#include "runtime/exec_env.h"
#include "runtime/thread_context.h"
+#include "storage/adaptive_thread_pool_controller.h"
+#include "util/threadpool.h"
namespace doris {
@@ -86,8 +88,18 @@ Status CloudDeltaWriter::write(const Block* block, const
DorisVector<uint32_t>&
CHECK(_is_init || _is_cancelled);
{
SCOPED_TIMER(_wait_flush_limit_timer);
- while (_memtable_writer->flush_running_count() >=
- config::memtable_flush_running_count_limit) {
+ auto* s3_file_upload_pool = rowset_builder()->is_s3_storage()
+ ?
ExecEnv::GetInstance()->s3_file_upload_thread_pool()
+ : nullptr;
+ const auto need_backpressure = [this, s3_file_upload_pool] {
+ if (s3_file_upload_pool != nullptr) {
+ return s3_file_upload_pool->get_queue_size() >
+ AdaptiveThreadPoolController::kS3QueueBusyThreshold;
+ }
+ return _memtable_writer->flush_running_count() >=
+ config::memtable_flush_running_count_limit;
+ };
+ while (need_backpressure()) {
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
}
diff --git a/be/src/cloud/cloud_rowset_builder.cpp
b/be/src/cloud/cloud_rowset_builder.cpp
index 9762f2b62b6..e03d347d94f 100644
--- a/be/src/cloud/cloud_rowset_builder.cpp
+++ b/be/src/cloud/cloud_rowset_builder.cpp
@@ -21,6 +21,7 @@
#include "cloud/cloud_storage_engine.h"
#include "cloud/cloud_tablet.h"
#include "cloud/cloud_tablet_mgr.h"
+#include "io/fs/file_system.h"
#include "storage/storage_policy.h"
namespace doris {
@@ -135,6 +136,13 @@ const RowsetMetaSharedPtr&
CloudRowsetBuilder::rowset_meta() {
return _rowset_writer->rowset_meta();
}
+bool CloudRowsetBuilder::is_s3_storage() const {
+ if (_rowset_writer == nullptr) {
+ return false;
+ }
+ return _rowset_writer->context().fs()->type() == io::FileSystemType::S3;
+}
+
Status CloudRowsetBuilder::set_txn_related_info() {
if (_tablet->enable_unique_key_merge_on_write()) {
// For empty rowsets when skip_writing_empty_rowset_metadata=true,
diff --git a/be/src/cloud/cloud_rowset_builder.h
b/be/src/cloud/cloud_rowset_builder.h
index cec8cfed979..5549a261659 100644
--- a/be/src/cloud/cloud_rowset_builder.h
+++ b/be/src/cloud/cloud_rowset_builder.h
@@ -37,6 +37,8 @@ public:
const RowsetMetaSharedPtr& rowset_meta();
+ bool is_s3_storage() const;
+
Status set_txn_related_info();
void set_skip_writing_rowset_metadata(bool skip) {
_skip_writing_rowset_metadata = skip; }
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
