This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 05ec0ba2e59 [fix](inverted_index) donot use int32_t for index id to
avoid overflow (#35062)
05ec0ba2e59 is described below
commit 05ec0ba2e59a7523fae009b1db3c3cfca3b1e099
Author: Yongqiang YANG <[email protected]>
AuthorDate: Mon May 20 20:05:23 2024 +0800
[fix](inverted_index) donot use int32_t for index id to avoid overflow
(#35062)
---
be/src/olap/rowset/beta_rowset.cpp | 2 +-
be/src/olap/rowset/beta_rowset.h | 2 +-
be/src/olap/rowset/rowset.h | 2 +-
be/src/olap/tablet.cpp | 2 +-
be/src/olap/tablet.h | 2 +-
be/src/olap/tablet_schema.cpp | 4 ++--
be/src/olap/tablet_schema.h | 4 ++--
be/src/olap/task/index_builder.cpp | 2 +-
be/src/olap/task/index_builder.h | 2 +-
be/test/testutil/mock_rowset.h | 2 +-
10 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/be/src/olap/rowset/beta_rowset.cpp
b/be/src/olap/rowset/beta_rowset.cpp
index 7587c15b36a..a1ac0d4539e 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -259,7 +259,7 @@ void BetaRowset::do_close() {
Status BetaRowset::link_files_to(const std::string& dir, RowsetId
new_rowset_id,
size_t new_rowset_start_seg_id,
- std::set<int32_t>* without_index_uids) {
+ std::set<int64_t>* without_index_uids) {
DCHECK(is_local());
auto fs = _rowset_meta->fs();
if (!fs) {
diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h
index af69e0c71c8..0fbfbda19d3 100644
--- a/be/src/olap/rowset/beta_rowset.h
+++ b/be/src/olap/rowset/beta_rowset.h
@@ -71,7 +71,7 @@ public:
Status link_files_to(const std::string& dir, RowsetId new_rowset_id,
size_t new_rowset_start_seg_id = 0,
- std::set<int32_t>* without_index_uids = nullptr)
override;
+ std::set<int64_t>* without_index_uids = nullptr)
override;
Status copy_files_to(const std::string& dir, const RowsetId&
new_rowset_id) override;
diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index 9364b46ba0b..8d1d20145e1 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -204,7 +204,7 @@ public:
// hard link all files in this rowset to `dir` to form a new rowset with
id `new_rowset_id`.
virtual Status link_files_to(const std::string& dir, RowsetId
new_rowset_id,
size_t new_rowset_start_seg_id = 0,
- std::set<int32_t>* without_index_uids =
nullptr) = 0;
+ std::set<int64_t>* without_index_uids =
nullptr) = 0;
// copy all files to `dir`
virtual Status copy_files_to(const std::string& dir, const RowsetId&
new_rowset_id) = 0;
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index f1e0b56d490..cb1e210cf68 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1191,7 +1191,7 @@ std::vector<RowsetSharedPtr>
Tablet::pick_candidate_rowsets_to_full_compaction()
}
std::vector<RowsetSharedPtr>
Tablet::pick_candidate_rowsets_to_build_inverted_index(
- const std::set<int32_t>& alter_index_uids, bool is_drop_op) {
+ const std::set<int64_t>& alter_index_uids, bool is_drop_op) {
std::vector<RowsetSharedPtr> candidate_rowsets;
{
std::shared_lock rlock(_meta_lock);
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 4f4faa01cb4..4f52927df75 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -267,7 +267,7 @@ public:
std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_base_compaction();
std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_full_compaction();
std::vector<RowsetSharedPtr>
pick_candidate_rowsets_to_build_inverted_index(
- const std::set<int32_t>& alter_index_uids, bool is_drop_op);
+ const std::set<int64_t>& alter_index_uids, bool is_drop_op);
// used for single compaction to get the local versions
// Single compaction does not require remote rowsets and cannot violate
the cooldown semantics
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 33e18986b10..11d3456653d 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -1295,7 +1295,7 @@ bool TabletSchema::has_inverted_index(const TabletColumn&
col) const {
return false;
}
-bool TabletSchema::has_inverted_index_with_index_id(int32_t index_id,
+bool TabletSchema::has_inverted_index_with_index_id(int64_t index_id,
const std::string&
suffix_name) const {
for (size_t i = 0; i < _indexes.size(); i++) {
if (_indexes[i].index_type() == IndexType::INVERTED &&
@@ -1307,7 +1307,7 @@ bool
TabletSchema::has_inverted_index_with_index_id(int32_t index_id,
}
const TabletIndex* TabletSchema::get_inverted_index_with_index_id(
- int32_t index_id, const std::string& suffix_name) const {
+ int64_t index_id, const std::string& suffix_name) const {
for (size_t i = 0; i < _indexes.size(); i++) {
if (_indexes[i].index_type() == IndexType::INVERTED &&
_indexes[i].get_index_suffix() == suffix_name &&
_indexes[i].index_id() == index_id) {
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index c4039369899..d08b95ae00f 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -346,8 +346,8 @@ public:
}
std::vector<const TabletIndex*> get_indexes_for_column(const TabletColumn&
col) const;
bool has_inverted_index(const TabletColumn& col) const;
- bool has_inverted_index_with_index_id(int32_t index_id, const std::string&
suffix_path) const;
- const TabletIndex* get_inverted_index_with_index_id(int32_t index_id,
+ bool has_inverted_index_with_index_id(int64_t index_id, const std::string&
suffix_path) const;
+ const TabletIndex* get_inverted_index_with_index_id(int64_t index_id,
const std::string&
suffix_name) const;
const TabletIndex* get_inverted_index(const TabletColumn& col) const;
const TabletIndex* get_inverted_index(int32_t col_unique_id,
diff --git a/be/src/olap/task/index_builder.cpp
b/be/src/olap/task/index_builder.cpp
index e7f2abac65c..8d2ba14b48f 100644
--- a/be/src/olap/task/index_builder.cpp
+++ b/be/src/olap/task/index_builder.cpp
@@ -61,7 +61,7 @@ Status IndexBuilder::update_inverted_index_info() {
LOG(INFO) << "begin to update_inverted_index_info, tablet=" <<
_tablet->tablet_id()
<< ", is_drop_op=" << _is_drop_op;
// index ids that will not be linked
- std::set<int32_t> without_index_uids;
+ std::set<int64_t> without_index_uids;
_output_rowsets.reserve(_input_rowsets.size());
_pending_rs_guards.reserve(_input_rowsets.size());
for (auto&& input_rowset : _input_rowsets) {
diff --git a/be/src/olap/task/index_builder.h b/be/src/olap/task/index_builder.h
index f2de2e06c6c..f9bc2b34d87 100644
--- a/be/src/olap/task/index_builder.h
+++ b/be/src/olap/task/index_builder.h
@@ -74,7 +74,7 @@ private:
std::vector<TColumn> _columns;
std::vector<doris::TOlapTableIndex> _alter_inverted_indexes;
bool _is_drop_op;
- std::set<int32_t> _alter_index_ids;
+ std::set<int64_t> _alter_index_ids;
std::vector<RowsetSharedPtr> _input_rowsets;
std::vector<RowsetSharedPtr> _output_rowsets;
std::vector<PendingRowsetGuard> _pending_rs_guards;
diff --git a/be/test/testutil/mock_rowset.h b/be/test/testutil/mock_rowset.h
index 89fbb8cac3b..461832a775a 100644
--- a/be/test/testutil/mock_rowset.h
+++ b/be/test/testutil/mock_rowset.h
@@ -30,7 +30,7 @@ class MockRowset : public Rowset {
Status remove() override { return Status::NotSupported("MockRowset not
support this method."); }
Status link_files_to(const std::string& dir, RowsetId new_rowset_id,
size_t start_seg_id,
- std::set<int32_t>* without_index_uids) override {
+ std::set<int64_t>* without_index_uids) override {
return Status::NotSupported("MockRowset not support this method.");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
