This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 6a5c2a7d0b3 [fix](schema change) fix schema change check does not
calculate reader merged rows (#33825) (#37467)
6a5c2a7d0b3 is described below
commit 6a5c2a7d0b397b097ca316a4db7f4b555ce87adb
Author: Luwei <[email protected]>
AuthorDate: Tue Jul 9 09:42:51 2024 +0800
[fix](schema change) fix schema change check does not calculate reader
merged rows (#33825) (#37467)
---
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 2 ++
be/src/olap/rowset/beta_rowset_reader.cpp | 3 +++
be/src/olap/rowset/beta_rowset_reader.h | 3 +++
be/src/olap/rowset/rowset_reader.h | 2 ++
be/src/olap/schema_change.h | 21 ++++++++++++++-------
6 files changed, 26 insertions(+), 7 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index c83f869095f..7fc4bc9c5d6 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1172,6 +1172,8 @@ DEFINE_mDouble(high_disk_avail_level_diff_usages, "0.15");
// create tablet in partition random robin idx lru size, default 10000
DEFINE_Int32(partition_disk_index_lru_size, "10000");
+DEFINE_mBool(ignore_schema_change_check, "false");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 8336515af3e..c762df19f53 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1225,6 +1225,8 @@ DECLARE_mDouble(high_disk_avail_level_diff_usages);
// create tablet in partition random robin idx lru size, default 10000
DECLARE_Int32(partition_disk_index_lru_size);
+DECLARE_mBool(ignore_schema_change_check);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp
b/be/src/olap/rowset/beta_rowset_reader.cpp
index 0ad45bebf3a..24a5f2d891f 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -273,6 +273,9 @@ Status BetaRowsetReader::_init_iterator() {
std::vector<RowwiseIteratorUPtr> iterators;
RETURN_IF_ERROR(get_segment_iterators(_read_context, &iterators));
+ if (_read_context->merged_rows == nullptr) {
+ _read_context->merged_rows = &_merged_rows;
+ }
// merge or union segment iterator
if (_is_merge_iterator()) {
auto sequence_loc = -1;
diff --git a/be/src/olap/rowset/beta_rowset_reader.h
b/be/src/olap/rowset/beta_rowset_reader.h
index db0b1facc73..336fb247dcf 100644
--- a/be/src/olap/rowset/beta_rowset_reader.h
+++ b/be/src/olap/rowset/beta_rowset_reader.h
@@ -72,6 +72,8 @@ public:
_stats->rows_vec_cond_filtered +
_stats->rows_short_circuit_cond_filtered;
}
+ uint64_t merged_rows() override { return *(_read_context->merged_rows); }
+
RowsetTypePB type() const override { return RowsetTypePB::BETA_ROWSET; }
Status current_block_row_locations(std::vector<RowLocation>* locations)
override {
@@ -125,6 +127,7 @@ private:
bool _empty = false;
size_t _topn_limit = 0;
+ uint64_t _merged_rows = 0;
};
} // namespace doris
diff --git a/be/src/olap/rowset/rowset_reader.h
b/be/src/olap/rowset/rowset_reader.h
index 34caa615c25..ead18c076ed 100644
--- a/be/src/olap/rowset/rowset_reader.h
+++ b/be/src/olap/rowset/rowset_reader.h
@@ -74,6 +74,8 @@ public:
virtual int64_t filtered_rows() = 0;
+ virtual uint64_t merged_rows() = 0;
+
virtual RowsetTypePB type() const = 0;
virtual int64_t newest_write_timestamp() = 0;
diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h
index c13f543d726..679a50b30b3 100644
--- a/be/src/olap/schema_change.h
+++ b/be/src/olap/schema_change.h
@@ -121,7 +121,6 @@ public:
RETURN_IF_ERROR(
_inner_process(rowset_reader, rowset_writer, new_tablet,
base_tablet_schema));
- _add_filtered_rows(rowset_reader->filtered_rows());
// Check row num changes
if (!_check_row_nums(rowset_reader, *rowset_writer)) {
@@ -129,8 +128,11 @@ public:
}
LOG(INFO) << "all row nums. source_rows=" <<
rowset_reader->rowset()->num_rows()
+ << ", source_filtered_rows=" <<
rowset_reader->filtered_rows()
+ << ", source_merged_rows=" << rowset_reader->merged_rows()
<< ", merged_rows=" << merged_rows() << ", filtered_rows="
<< filtered_rows()
- << ", new_index_rows=" << rowset_writer->num_rows();
+ << ", new_index_rows=" << rowset_writer->num_rows()
+ << ", writer_filtered_rows=" <<
rowset_writer->num_rows_filtered();
return Status::OK();
}
@@ -149,14 +151,19 @@ protected:
}
virtual bool _check_row_nums(RowsetReaderSharedPtr reader, const
RowsetWriter& writer) const {
- if (reader->rowset()->num_rows() != writer.num_rows() + _merged_rows +
_filtered_rows) {
+ if (reader->rowset()->num_rows() - reader->filtered_rows() -
reader->merged_rows() !=
+ writer.num_rows() + writer.num_rows_filtered() + _merged_rows +
_filtered_rows) {
LOG(WARNING) << "fail to check row num! "
<< "source_rows=" << reader->rowset()->num_rows()
- << ", writer rows=" << writer.num_rows()
+ << ", source_filtered_rows=" <<
reader->filtered_rows()
+ << ", source_merged_rows=" << reader->merged_rows()
+ << ", written_rows=" << writer.num_rows()
+ << ", writer_filtered_rows=" <<
writer.num_rows_filtered()
<< ", merged_rows=" << merged_rows()
- << ", filtered_rows=" << filtered_rows()
- << ", new_index_rows=" << writer.num_rows();
- return false;
+ << ", filtered_rows=" << filtered_rows();
+ if (!config::ignore_schema_change_check) {
+ return false;
+ }
}
return true;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
