This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 439027119e2 [fix](schema change) fix schema change check does not
calculate reader merged rows (#33825) (#33908)
439027119e2 is described below
commit 439027119e215a562ba157a1c4dd5d4659999841
Author: Luwei <[email protected]>
AuthorDate: Fri Apr 19 22:57:25 2024 +0800
[fix](schema change) fix schema change check does not calculate reader
merged rows (#33825) (#33908)
---
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 2 ++
be/src/olap/rowset/beta_rowset_reader.cpp | 3 +++
be/src/olap/rowset/beta_rowset_reader.h | 3 +++
be/src/olap/rowset/rowset_reader.h | 2 ++
be/src/olap/schema_change.h | 12 +++++++++---
6 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 07d24df34f6..57ed4e4933b 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1199,6 +1199,8 @@ DEFINE_mInt64(hive_sink_max_file_size, "1073741824"); //
1GB
DEFINE_mInt32(thrift_client_open_num_tries, "1");
+DEFINE_mBool(ignore_schema_change_check, "false");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 0c4d511dd99..6efb4d74cb8 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1281,6 +1281,8 @@ DECLARE_mInt64(hive_sink_max_file_size);
// Retry the Open num_retries time waiting 100 milliseconds between retries.
DECLARE_mInt32(thrift_client_open_num_tries);
+DECLARE_mBool(ignore_schema_change_check);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp
b/be/src/olap/rowset/beta_rowset_reader.cpp
index 80c3085ac7a..71729f0a230 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -297,6 +297,9 @@ Status BetaRowsetReader::_init_iterator() {
std::vector<RowwiseIteratorUPtr> iterators;
RETURN_IF_ERROR(get_segment_iterators(_read_context, &iterators));
+ if (_read_context->merged_rows == nullptr) {
+ _read_context->merged_rows = &_merged_rows;
+ }
// merge or union segment iterator
if (_is_merge_iterator()) {
auto sequence_loc = -1;
diff --git a/be/src/olap/rowset/beta_rowset_reader.h
b/be/src/olap/rowset/beta_rowset_reader.h
index ee23b2d9b9f..6d0c4034c87 100644
--- a/be/src/olap/rowset/beta_rowset_reader.h
+++ b/be/src/olap/rowset/beta_rowset_reader.h
@@ -72,6 +72,8 @@ public:
_stats->rows_vec_cond_filtered +
_stats->rows_short_circuit_cond_filtered;
}
+ uint64_t merged_rows() override { return *(_read_context->merged_rows); }
+
RowsetTypePB type() const override { return RowsetTypePB::BETA_ROWSET; }
Status current_block_row_locations(std::vector<RowLocation>* locations)
override {
@@ -126,6 +128,7 @@ private:
bool _empty = false;
size_t _topn_limit = 0;
+ uint64_t _merged_rows = 0;
};
} // namespace doris
diff --git a/be/src/olap/rowset/rowset_reader.h
b/be/src/olap/rowset/rowset_reader.h
index 9ada1706d89..58c0f592b9c 100644
--- a/be/src/olap/rowset/rowset_reader.h
+++ b/be/src/olap/rowset/rowset_reader.h
@@ -74,6 +74,8 @@ public:
virtual int64_t filtered_rows() = 0;
+ virtual uint64_t merged_rows() = 0;
+
virtual RowsetTypePB type() const = 0;
virtual int64_t newest_write_timestamp() = 0;
diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h
index 949c1d5514c..6528b587a9a 100644
--- a/be/src/olap/schema_change.h
+++ b/be/src/olap/schema_change.h
@@ -125,8 +125,11 @@ public:
}
LOG(INFO) << "all row nums. source_rows=" <<
rowset_reader->rowset()->num_rows()
+ << ", source_filtered_rows=" <<
rowset_reader->filtered_rows()
+ << ", source_merged_rows=" << rowset_reader->merged_rows()
<< ", merged_rows=" << merged_rows() << ", filtered_rows="
<< filtered_rows()
- << ", new_index_rows=" << rowset_writer->num_rows();
+ << ", new_index_rows=" << rowset_writer->num_rows()
+ << ", writer_filtered_rows=" <<
rowset_writer->num_rows_filtered();
return Status::OK();
}
@@ -146,16 +149,19 @@ protected:
}
virtual bool _check_row_nums(RowsetReaderSharedPtr reader, const
RowsetWriter& writer) const {
- if (reader->rowset()->num_rows() - reader->filtered_rows() !=
+ if (reader->rowset()->num_rows() - reader->filtered_rows() -
reader->merged_rows() !=
writer.num_rows() + writer.num_rows_filtered() + _merged_rows +
_filtered_rows) {
LOG(WARNING) << "fail to check row num! "
<< "source_rows=" << reader->rowset()->num_rows()
<< ", source_filtered_rows=" <<
reader->filtered_rows()
+ << ", source_merged_rows=" << reader->merged_rows()
<< ", written_rows=" << writer.num_rows()
<< ", writer_filtered_rows=" <<
writer.num_rows_filtered()
<< ", merged_rows=" << merged_rows()
<< ", filtered_rows=" << filtered_rows();
- return false;
+ if (!config::ignore_schema_change_check) {
+ return false;
+ }
}
return true;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
