This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a9b5cf2c612 [fix](spill) fix spill data usage counter is not update
correctly (#37352)
a9b5cf2c612 is described below
commit a9b5cf2c61213e6159f404863dfdee23e34385ba
Author: TengJianPing <[email protected]>
AuthorDate: Tue Jul 16 11:28:05 2024 +0800
[fix](spill) fix spill data usage counter is not update correctly (#37352)
## Proposed changes
Issue Number: close #xxx
If QueryContext is destructed earlier than PipelineFragmentContext,
spill_dir_ will be already moved to spill_gc directory, which result int
spill data disk usage is not decreased.
---
be/src/vec/spill/spill_stream.cpp | 11 +++++++----
be/src/vec/spill/spill_writer.cpp | 22 +++++++++++-----------
2 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/be/src/vec/spill/spill_stream.cpp
b/be/src/vec/spill/spill_stream.cpp
index 110a9f01c03..5ef0495432f 100644
--- a/be/src/vec/spill/spill_stream.cpp
+++ b/be/src/vec/spill/spill_stream.cpp
@@ -68,11 +68,14 @@ void SpillStream::gc() {
LOG_EVERY_T(WARNING, 1) << fmt::format("failed to gc spill data,
dir {}, error: {}",
query_gc_dir,
status.to_string());
}
- // decrease spill data usage anyway, since in ~QueryContext() spill
data of the query will be
- // clean up as a last resort
- data_dir_->update_spill_data_usage(-total_written_bytes_);
- total_written_bytes_ = 0;
}
+ // If QueryContext is destructed earlier than PipelineFragmentContext,
+ // spill_dir_ will be already moved to spill_gc directory.
+
+ // decrease spill data usage anyway, since in ~QueryContext() spill data
of the query will be
+ // clean up as a last resort
+ data_dir_->update_spill_data_usage(-total_written_bytes_);
+ total_written_bytes_ = 0;
}
Status SpillStream::prepare() {
diff --git a/be/src/vec/spill/spill_writer.cpp
b/be/src/vec/spill/spill_writer.cpp
index a48fb0b1dcb..46a97285802 100644
--- a/be/src/vec/spill/spill_writer.cpp
+++ b/be/src/vec/spill/spill_writer.cpp
@@ -122,9 +122,19 @@ Status SpillWriter::_write_internal(const Block& block,
size_t& written_bytes) {
}
{
+ auto buff_size = buff.size();
Defer defer {[&]() {
if (status.ok()) {
- data_dir_->update_spill_data_usage(buff.size());
+ data_dir_->update_spill_data_usage(buff_size);
+
+ written_bytes += buff_size;
+ max_sub_block_size_ = std::max(max_sub_block_size_,
buff_size);
+
+ meta_.append((const char*)&total_written_bytes_,
sizeof(size_t));
+ COUNTER_UPDATE(write_bytes_counter_, buff_size);
+ COUNTER_UPDATE(write_block_counter_, 1);
+ total_written_bytes_ += buff_size;
+ ++written_blocks_;
}
}};
{
@@ -135,16 +145,6 @@ Status SpillWriter::_write_internal(const Block& block,
size_t& written_bytes) {
}
}
- auto buff_size = buff.size();
- written_bytes += buff_size;
- max_sub_block_size_ = std::max(max_sub_block_size_, buff_size);
-
- meta_.append((const char*)&total_written_bytes_, sizeof(size_t));
- COUNTER_UPDATE(write_bytes_counter_, buff_size);
- COUNTER_UPDATE(write_block_counter_, 1);
- total_written_bytes_ += buff_size;
- ++written_blocks_;
-
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
