This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new db69a6d4722 [fix](inverted_index) donot use int32_t for index id to
avoid overflow (#35572)
db69a6d4722 is described below
commit db69a6d47229c3af80afb0f11a8b3593eb5c802a
Author: walter <[email protected]>
AuthorDate: Thu May 30 09:43:39 2024 +0800
[fix](inverted_index) donot use int32_t for index id to avoid overflow
(#35572)
Cherry-pick #35062
Co-authored-by: Yongqiang YANG
<[email protected]>
---
be/src/olap/rowset/beta_rowset.cpp | 2 +-
be/src/olap/rowset/beta_rowset.h | 2 +-
be/src/olap/rowset/rowset.h | 2 +-
be/src/olap/tablet.cpp | 2 +-
be/src/olap/tablet.h | 2 +-
be/src/olap/tablet_schema.cpp | 2 +-
be/src/olap/tablet_schema.h | 2 +-
be/src/olap/task/index_builder.cpp | 2 +-
be/src/olap/task/index_builder.h | 2 +-
be/test/testutil/mock_rowset.h | 2 +-
10 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/be/src/olap/rowset/beta_rowset.cpp
b/be/src/olap/rowset/beta_rowset.cpp
index bf3cb1d1e74..6aea5d3d98b 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -233,7 +233,7 @@ void BetaRowset::do_close() {
Status BetaRowset::link_files_to(const std::string& dir, RowsetId
new_rowset_id,
size_t new_rowset_start_seg_id,
- std::set<int32_t>* without_index_uids) {
+ std::set<int64_t>* without_index_uids) {
DCHECK(is_local());
auto fs = _rowset_meta->fs();
if (!fs) {
diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h
index f4496457183..ed30f76d450 100644
--- a/be/src/olap/rowset/beta_rowset.h
+++ b/be/src/olap/rowset/beta_rowset.h
@@ -72,7 +72,7 @@ public:
Status link_files_to(const std::string& dir, RowsetId new_rowset_id,
size_t new_rowset_start_seg_id = 0,
- std::set<int32_t>* without_index_uids = nullptr)
override;
+ std::set<int64_t>* without_index_uids = nullptr)
override;
Status copy_files_to(const std::string& dir, const RowsetId&
new_rowset_id) override;
diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index c860cf6f646..b1566387144 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -206,7 +206,7 @@ public:
// hard link all files in this rowset to `dir` to form a new rowset with
id `new_rowset_id`.
virtual Status link_files_to(const std::string& dir, RowsetId
new_rowset_id,
size_t new_rowset_start_seg_id = 0,
- std::set<int32_t>* without_index_uids =
nullptr) = 0;
+ std::set<int64_t>* without_index_uids =
nullptr) = 0;
// copy all files to `dir`
virtual Status copy_files_to(const std::string& dir, const RowsetId&
new_rowset_id) = 0;
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index db635554b00..b6fb72b434f 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1415,7 +1415,7 @@ std::vector<RowsetSharedPtr>
Tablet::pick_first_consecutive_empty_rowsets(int li
}
std::vector<RowsetSharedPtr>
Tablet::pick_candidate_rowsets_to_build_inverted_index(
- const std::set<int32_t>& alter_index_uids, bool is_drop_op) {
+ const std::set<int64_t>& alter_index_uids, bool is_drop_op) {
std::vector<RowsetSharedPtr> candidate_rowsets;
{
std::shared_lock rlock(_meta_lock);
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 45694e30602..c790c3330bf 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -280,7 +280,7 @@ public:
std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_base_compaction();
std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_full_compaction();
std::vector<RowsetSharedPtr>
pick_candidate_rowsets_to_build_inverted_index(
- const std::set<int32_t>& alter_index_uids, bool is_drop_op);
+ const std::set<int64_t>& alter_index_uids, bool is_drop_op);
// used for single compaction to get the local versions
// Single compaction does not require remote rowsets and cannot violate
the cooldown semantics
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 599cfb39d45..6dd04782588 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -984,7 +984,7 @@ bool TabletSchema::has_inverted_index(int32_t
col_unique_id) const {
return false;
}
-bool TabletSchema::has_inverted_index_with_index_id(int32_t index_id) const {
+bool TabletSchema::has_inverted_index_with_index_id(int64_t index_id) const {
for (size_t i = 0; i < _indexes.size(); i++) {
if (_indexes[i].index_type() == IndexType::INVERTED &&
_indexes[i].index_id() == index_id) {
return true;
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 0495b2c9460..2ffbfb0ffe8 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -271,7 +271,7 @@ public:
const std::vector<TabletIndex>& indexes() const { return _indexes; }
std::vector<const TabletIndex*> get_indexes_for_column(int32_t
col_unique_id) const;
bool has_inverted_index(int32_t col_unique_id) const;
- bool has_inverted_index_with_index_id(int32_t index_id) const;
+ bool has_inverted_index_with_index_id(int64_t index_id) const;
const TabletIndex* get_inverted_index(int32_t col_unique_id) const;
bool has_ngram_bf_index(int32_t col_unique_id) const;
const TabletIndex* get_ngram_bf_index(int32_t col_unique_id) const;
diff --git a/be/src/olap/task/index_builder.cpp
b/be/src/olap/task/index_builder.cpp
index be41f8b3706..7f2f6af3740 100644
--- a/be/src/olap/task/index_builder.cpp
+++ b/be/src/olap/task/index_builder.cpp
@@ -58,7 +58,7 @@ Status IndexBuilder::update_inverted_index_info() {
LOG(INFO) << "begin to update_inverted_index_info, tablet=" <<
_tablet->tablet_id()
<< ", is_drop_op=" << _is_drop_op;
// index ids that will not be linked
- std::set<int32_t> without_index_uids;
+ std::set<int64_t> without_index_uids;
for (auto i = 0; i < _input_rowsets.size(); ++i) {
auto input_rowset = _input_rowsets[i];
TabletSchemaSPtr output_rs_tablet_schema =
std::make_shared<TabletSchema>();
diff --git a/be/src/olap/task/index_builder.h b/be/src/olap/task/index_builder.h
index 2ff266f5a77..41d87e388f9 100644
--- a/be/src/olap/task/index_builder.h
+++ b/be/src/olap/task/index_builder.h
@@ -64,7 +64,7 @@ private:
std::vector<TColumn> _columns;
std::vector<doris::TOlapTableIndex> _alter_inverted_indexes;
bool _is_drop_op;
- std::set<int32_t> _alter_index_ids;
+ std::set<int64_t> _alter_index_ids;
std::vector<RowsetSharedPtr> _input_rowsets;
std::vector<RowsetSharedPtr> _output_rowsets;
std::vector<RowsetReaderSharedPtr> _input_rs_readers;
diff --git a/be/test/testutil/mock_rowset.h b/be/test/testutil/mock_rowset.h
index 93b76d9c67b..def5c4a9661 100644
--- a/be/test/testutil/mock_rowset.h
+++ b/be/test/testutil/mock_rowset.h
@@ -30,7 +30,7 @@ class MockRowset : public Rowset {
Status remove() override { return Status::NotSupported("MockRowset not
support this method."); }
Status link_files_to(const std::string& dir, RowsetId new_rowset_id,
size_t start_seg_id,
- std::set<int32_t>* without_index_uids) override {
+ std::set<int64_t>* without_index_uids) override {
return Status::NotSupported("MockRowset not support this method.");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
