Gabriel39 commented on code in PR #66825:
URL: https://github.com/apache/doris/pull/66825#discussion_r3841145373
##########
be/src/exec/sink/writer/iceberg/viceberg_sort_writer.cpp:
##########
@@ -85,6 +87,59 @@ size_t
VIcebergSortWriter::get_reserve_mem_size(RuntimeState* state, bool eos) c
return _sorter == nullptr ? 0 : _sorter->get_reserve_mem_size(state, eos);
}
+SorterReserveMemory
VIcebergSortWriter::get_reserve_mem_size_components(RuntimeState* state,
+ bool
eos) const {
+ std::lock_guard<std::mutex> lock(_sorter_mutex);
+ if (_sorter == nullptr) {
+ return {};
+ }
+ const size_t target = _target_file_size_bytes >= 0
+ ?
static_cast<size_t>(_target_file_size_bytes)
+ : std::numeric_limits<size_t>::max();
+ auto reservation = _sorter->get_reserve_mem_size_components(state, eos,
target);
+ _include_merge_reservation(state, eos, &reservation);
+ return reservation;
+}
+
+SorterReserveMemory VIcebergSortWriter::get_reserve_mem_size_components(
+ RuntimeState* state, bool eos, size_t incoming_rows, size_t
incoming_bytes) const {
+ std::lock_guard<std::mutex> lock(_sorter_mutex);
+ if (_sorter == nullptr) {
+ return {};
+ }
+ const size_t target = _target_file_size_bytes >= 0
+ ?
static_cast<size_t>(_target_file_size_bytes)
+ : std::numeric_limits<size_t>::max();
+ auto reservation = _sorter->get_reserve_mem_size_components(state, eos,
incoming_rows,
+
incoming_bytes, target);
+ _include_merge_reservation(state, eos, &reservation);
+ return reservation;
+}
+
+void VIcebergSortWriter::_include_merge_reservation(RuntimeState* state, bool
eos,
+ SorterReserveMemory*
reservation) const {
+ if (!eos) {
Review Comment:
Fixed in 643370c0f9. Non-EOS admission now detects a predicted target-file
rollover from the current sorter data plus incoming bytes and includes the
byte-bounded merge-output workspace. The regression test retains a sorted run
from wide rows and uses a tiny tail to cross the target.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
