This is an automated email from the ASF dual-hosted git repository. dataroaring pushed a commit to branch branch-4.0-preview in repository https://gitbox.apache.org/repos/asf/doris.git
commit 351714a143d87eaefd1d30ba85e401716274a251 Author: Luwei <[email protected]> AuthorDate: Fri Apr 19 15:17:39 2024 +0800 [fix](schema change) fix schema change check does not calculate reader merged rows (#33825) * [fix](schema change) fix schema change verification does not calculate reader merged rows * fix code style * fix --- be/src/common/config.cpp | 2 ++ be/src/common/config.h | 2 ++ be/src/olap/rowset/beta_rowset_reader.cpp | 3 +++ be/src/olap/rowset/beta_rowset_reader.h | 3 +++ be/src/olap/rowset/rowset_reader.h | 2 ++ be/src/olap/schema_change.h | 12 +++++++++--- 6 files changed, 21 insertions(+), 3 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 988cf3b53d6..e791a902704 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1203,6 +1203,8 @@ DEFINE_Bool(enable_index_compaction, "false"); // enable injection point in regression-test DEFINE_mBool(enable_injection_point, "false"); +DEFINE_mBool(ignore_schema_change_check, "false"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index 10228fbfbd7..1e2f71c452b 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1292,6 +1292,8 @@ DECLARE_mInt32(thrift_client_open_num_tries); // enable injection point in regression-test DECLARE_mBool(enable_injection_point); +DECLARE_mBool(ignore_schema_change_check); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index cca7765a610..1ec4f2bbb10 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -305,6 +305,9 @@ Status BetaRowsetReader::_init_iterator() { std::vector<RowwiseIteratorUPtr> iterators; RETURN_IF_ERROR(get_segment_iterators(_read_context, &iterators)); + if (_read_context->merged_rows == nullptr) { + _read_context->merged_rows = &_merged_rows; + } // merge or union segment iterator if (_is_merge_iterator()) { auto sequence_loc = -1; diff --git a/be/src/olap/rowset/beta_rowset_reader.h b/be/src/olap/rowset/beta_rowset_reader.h index ee23b2d9b9f..6d0c4034c87 100644 --- a/be/src/olap/rowset/beta_rowset_reader.h +++ b/be/src/olap/rowset/beta_rowset_reader.h @@ -72,6 +72,8 @@ public: _stats->rows_vec_cond_filtered + _stats->rows_short_circuit_cond_filtered; } + uint64_t merged_rows() override { return *(_read_context->merged_rows); } + RowsetTypePB type() const override { return RowsetTypePB::BETA_ROWSET; } Status current_block_row_locations(std::vector<RowLocation>* locations) override { @@ -126,6 +128,7 @@ private: bool _empty = false; size_t _topn_limit = 0; + uint64_t _merged_rows = 0; }; } // namespace doris diff --git a/be/src/olap/rowset/rowset_reader.h b/be/src/olap/rowset/rowset_reader.h index 9ada1706d89..58c0f592b9c 100644 --- a/be/src/olap/rowset/rowset_reader.h +++ b/be/src/olap/rowset/rowset_reader.h @@ -74,6 +74,8 @@ public: virtual int64_t filtered_rows() = 0; + virtual uint64_t merged_rows() = 0; + virtual RowsetTypePB type() const = 0; virtual int64_t newest_write_timestamp() = 0; diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h index c3c6bf7e9ab..9d67bdbf5ee 100644 --- a/be/src/olap/schema_change.h +++ b/be/src/olap/schema_change.h @@ -126,8 +126,11 @@ public: } LOG(INFO) << "all row nums. source_rows=" << rowset_reader->rowset()->num_rows() + << ", source_filtered_rows=" << rowset_reader->filtered_rows() + << ", source_merged_rows=" << rowset_reader->merged_rows() << ", merged_rows=" << merged_rows() << ", filtered_rows=" << filtered_rows() - << ", new_index_rows=" << rowset_writer->num_rows(); + << ", new_index_rows=" << rowset_writer->num_rows() + << ", writer_filtered_rows=" << rowset_writer->num_rows_filtered(); return Status::OK(); } @@ -147,16 +150,19 @@ protected: } virtual bool _check_row_nums(RowsetReaderSharedPtr reader, const RowsetWriter& writer) const { - if (reader->rowset()->num_rows() - reader->filtered_rows() != + if (reader->rowset()->num_rows() - reader->filtered_rows() - reader->merged_rows() != writer.num_rows() + writer.num_rows_filtered() + _merged_rows + _filtered_rows) { LOG(WARNING) << "fail to check row num! " << "source_rows=" << reader->rowset()->num_rows() << ", source_filtered_rows=" << reader->filtered_rows() + << ", source_merged_rows=" << reader->merged_rows() << ", written_rows=" << writer.num_rows() << ", writer_filtered_rows=" << writer.num_rows_filtered() << ", merged_rows=" << merged_rows() << ", filtered_rows=" << filtered_rows(); - return false; + if (!config::ignore_schema_change_check) { + return false; + } } return true; } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
