gavinchou commented on code in PR #66847:
URL: https://github.com/apache/doris/pull/66847#discussion_r3809839605
##########
be/src/cloud/cloud_delta_writer.cpp:
##########
@@ -99,9 +101,30 @@ Status CloudDeltaWriter::write(const Block* block, const
TabletAddRowsPayload& r
CHECK(_is_init || _is_cancelled);
{
SCOPED_TIMER(_wait_flush_limit_timer);
- while (_memtable_writer->flush_running_count() >=
- config::memtable_flush_running_count_limit) {
+ const auto effective_flush_running_count_limit =
+ config::memtable_flush_running_count_limit *
+ (_req.write_req_type == WriteRequestType::GROUP ? 2 : 1);
+ const bool use_s3_upload_queue = rowset_builder()->is_s3_storage();
+ auto* s3_file_upload_pool = use_s3_upload_queue
+ ?
ExecEnv::GetInstance()->s3_file_upload_thread_pool()
+ : nullptr;
+ bool need_back_press = false;
+ if (use_s3_upload_queue && s3_file_upload_pool != nullptr) {
+ need_back_press = s3_file_upload_pool->get_queue_size() >
+
AdaptiveThreadPoolController::kS3QueueBusyThreshold;
+ } else {
+ need_back_press =
+ _memtable_writer->flush_running_count() >=
effective_flush_running_count_limit;
+ }
+ while (need_back_press) {
std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ if (use_s3_upload_queue && s3_file_upload_pool != nullptr) {
+ need_back_press = s3_file_upload_pool->get_queue_size() >
+
AdaptiveThreadPoolController::kS3QueueBusyThreshold;
+ } else {
+ need_back_press = _memtable_writer->flush_running_count() >=
+ effective_flush_running_count_limit;
+ }
Review Comment:
verbosy check, e.g. need_back_press
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
