This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-4.2 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 47487e1ddd7d896f8ef72120f90bb0f4219e2cd6 Author: hui lai <[email protected]> AuthorDate: Thu Sep 10 09:28:03 2026 +0800 branch-4.1: [perf](load) Optimize write performance with improved backpressure control (#67728) ### What problem does this PR solve? Backport #66847 to branch-4.1. For S3-backed cloud loads, a memtable flush can finish after submitting asynchronous uploads, so flush-running count does not represent downstream upload pressure. This change detects S3 storage and uses the S3 upload queue for backpressure, falling back to the existing flush-running limit when the upload pool is unavailable. HDFS retains flush-count backpressure. ### Branch-4.1 adaptation branch-4.1 does not contain WriteRequestType::GROUP or CloudGroupRowsetBuilder. The row-binlog GROUP-specific changes from the original PR are therefore omitted. CloudRowsetBuilder is final in this branch, so is_s3_storage is non-virtual. Local-storage flush limits remain unchanged. ### Release note Improve load backpressure for S3-backed cloud loads. Higher concurrency can increase memory use. ### Check List (For Author) - Test: Compilation and tests skipped as requested. clang-format 16 and git diff --check passed. - Behavior changed: Yes; S3-backed loads use upload queue pressure. HDFS and local-storage limits remain unchanged. - Does this need documentation: No. --- be/src/cloud/cloud_delta_writer.cpp | 16 ++++++++++++++-- be/src/cloud/cloud_rowset_builder.cpp | 8 ++++++++ be/src/cloud/cloud_rowset_builder.h | 2 ++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/be/src/cloud/cloud_delta_writer.cpp b/be/src/cloud/cloud_delta_writer.cpp index 7a7fd00bb68..b85e83be20c 100644 --- a/be/src/cloud/cloud_delta_writer.cpp +++ b/be/src/cloud/cloud_delta_writer.cpp @@ -25,6 +25,8 @@ #include "load/memtable/memtable_memory_limiter.h" #include "runtime/exec_env.h" #include "runtime/thread_context.h" +#include "storage/adaptive_thread_pool_controller.h" +#include "util/threadpool.h" namespace doris { @@ -86,8 +88,18 @@ Status CloudDeltaWriter::write(const Block* block, const DorisVector<uint32_t>& CHECK(_is_init || _is_cancelled); { SCOPED_TIMER(_wait_flush_limit_timer); - while (_memtable_writer->flush_running_count() >= - config::memtable_flush_running_count_limit) { + auto* s3_file_upload_pool = rowset_builder()->is_s3_storage() + ? ExecEnv::GetInstance()->s3_file_upload_thread_pool() + : nullptr; + const auto need_backpressure = [this, s3_file_upload_pool] { + if (s3_file_upload_pool != nullptr) { + return s3_file_upload_pool->get_queue_size() > + AdaptiveThreadPoolController::kS3QueueBusyThreshold; + } + return _memtable_writer->flush_running_count() >= + config::memtable_flush_running_count_limit; + }; + while (need_backpressure()) { std::this_thread::sleep_for(std::chrono::milliseconds(10)); } } diff --git a/be/src/cloud/cloud_rowset_builder.cpp b/be/src/cloud/cloud_rowset_builder.cpp index 9762f2b62b6..e03d347d94f 100644 --- a/be/src/cloud/cloud_rowset_builder.cpp +++ b/be/src/cloud/cloud_rowset_builder.cpp @@ -21,6 +21,7 @@ #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" #include "cloud/cloud_tablet_mgr.h" +#include "io/fs/file_system.h" #include "storage/storage_policy.h" namespace doris { @@ -135,6 +136,13 @@ const RowsetMetaSharedPtr& CloudRowsetBuilder::rowset_meta() { return _rowset_writer->rowset_meta(); } +bool CloudRowsetBuilder::is_s3_storage() const { + if (_rowset_writer == nullptr) { + return false; + } + return _rowset_writer->context().fs()->type() == io::FileSystemType::S3; +} + Status CloudRowsetBuilder::set_txn_related_info() { if (_tablet->enable_unique_key_merge_on_write()) { // For empty rowsets when skip_writing_empty_rowset_metadata=true, diff --git a/be/src/cloud/cloud_rowset_builder.h b/be/src/cloud/cloud_rowset_builder.h index cec8cfed979..5549a261659 100644 --- a/be/src/cloud/cloud_rowset_builder.h +++ b/be/src/cloud/cloud_rowset_builder.h @@ -37,6 +37,8 @@ public: const RowsetMetaSharedPtr& rowset_meta(); + bool is_s3_storage() const; + Status set_txn_related_info(); void set_skip_writing_rowset_metadata(bool skip) { _skip_writing_rowset_metadata = skip; } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
