This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new b66ddbbea78 branch-3.1: [fix](compaction) fix vertical compaction IO
stats accumulation #58756 (#58865)
b66ddbbea78 is described below
commit b66ddbbea78ca6ec0405893678fed1e2d514de00
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Dec 15 11:14:11 2025 +0800
branch-3.1: [fix](compaction) fix vertical compaction IO stats accumulation
#58756 (#58865)
Cherry-picked from #58756
Co-authored-by: Luwei <[email protected]>
---
be/src/olap/merger.cpp | 46 +++++++++++++++++++++++++++++++++++++---------
1 file changed, 37 insertions(+), 9 deletions(-)
diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp
index e6c2d9a4334..d5c435868e9 100644
--- a/be/src/olap/merger.cpp
+++ b/be/src/olap/merger.cpp
@@ -319,10 +319,12 @@ Status Merger::vertical_compact_one_group(
tablet->tablet_id());
}
- if (is_key && stats_output != nullptr) {
- stats_output->output_rows = output_rows;
- stats_output->merged_rows = reader.merged_rows();
- stats_output->filtered_rows = reader.filtered_rows();
+ if (stats_output != nullptr) {
+ if (is_key) {
+ stats_output->output_rows = output_rows;
+ stats_output->merged_rows = reader.merged_rows();
+ stats_output->filtered_rows = reader.filtered_rows();
+ }
stats_output->bytes_read_from_local =
reader.stats().file_cache_stats.bytes_read_from_local;
stats_output->bytes_read_from_remote =
reader.stats().file_cache_stats.bytes_read_from_remote;
@@ -373,10 +375,12 @@ Status Merger::vertical_compact_one_group(
tablet_id);
}
- if (is_key && stats_output != nullptr) {
- stats_output->output_rows = output_rows;
- stats_output->merged_rows = src_block_reader.merged_rows();
- stats_output->filtered_rows = src_block_reader.filtered_rows();
+ if (stats_output != nullptr) {
+ if (is_key) {
+ stats_output->output_rows = output_rows;
+ stats_output->merged_rows = src_block_reader.merged_rows();
+ stats_output->filtered_rows = src_block_reader.filtered_rows();
+ }
stats_output->bytes_read_from_local =
src_block_reader.stats().file_cache_stats.bytes_read_from_local;
stats_output->bytes_read_from_remote =
@@ -466,6 +470,10 @@ Status Merger::vertical_merge_rowsets(BaseTabletSPtr
tablet, ReaderType reader_t
vectorized::RowSourcesBuffer row_sources_buf(
tablet->tablet_id(), dst_rowset_writer->context().tablet_path,
reader_type);
+ Merger::Statistics total_stats;
+ if (stats_output != nullptr) {
+ total_stats.rowid_conversion = stats_output->rowid_conversion;
+ }
{
std::unique_lock<std::mutex> lock(tablet->sample_info_lock);
tablet->sample_infos.resize(column_groups.size(), {0, 0, 0});
@@ -478,15 +486,31 @@ Status Merger::vertical_merge_rowsets(BaseTabletSPtr
tablet, ReaderType reader_t
? config::compaction_batch_size
: estimate_batch_size(i, tablet,
merge_way_num);
CompactionSampleInfo sample_info;
+ Merger::Statistics group_stats;
+ group_stats.rowid_conversion = total_stats.rowid_conversion;
+ Merger::Statistics* group_stats_ptr = stats_output != nullptr ?
&group_stats : nullptr;
Status st = vertical_compact_one_group(
tablet, reader_type, tablet_schema, is_key, column_groups[i],
&row_sources_buf,
- src_rowset_readers, dst_rowset_writer, max_rows_per_segment,
stats_output,
+ src_rowset_readers, dst_rowset_writer, max_rows_per_segment,
group_stats_ptr,
key_group_cluster_key_idxes, batch_size, &sample_info);
{
std::unique_lock<std::mutex> lock(tablet->sample_info_lock);
tablet->sample_infos[i] = sample_info;
}
RETURN_IF_ERROR(st);
+ if (stats_output != nullptr) {
+ total_stats.bytes_read_from_local +=
group_stats.bytes_read_from_local;
+ total_stats.bytes_read_from_remote +=
group_stats.bytes_read_from_remote;
+ total_stats.cached_bytes_total += group_stats.cached_bytes_total;
+ total_stats.cloud_local_read_time +=
group_stats.cloud_local_read_time;
+ total_stats.cloud_remote_read_time +=
group_stats.cloud_remote_read_time;
+ if (is_key) {
+ total_stats.output_rows = group_stats.output_rows;
+ total_stats.merged_rows = group_stats.merged_rows;
+ total_stats.filtered_rows = group_stats.filtered_rows;
+ total_stats.rowid_conversion = group_stats.rowid_conversion;
+ }
+ }
if (is_key) {
RETURN_IF_ERROR(row_sources_buf.flush());
}
@@ -497,6 +521,10 @@ Status Merger::vertical_merge_rowsets(BaseTabletSPtr
tablet, ReaderType reader_t
VLOG_NOTICE << "finish compact groups";
RETURN_IF_ERROR(dst_rowset_writer->final_flush());
+ if (stats_output != nullptr) {
+ *stats_output = total_stats;
+ }
+
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
