This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 8321a3f81f6 [fix](inverted index)Change index_id from int32 to int64
to avoid overflow #35206 (#35573)
8321a3f81f6 is described below
commit 8321a3f81f67314f46fe7f337aa560aaf7cc0fe3
Author: walter <[email protected]>
AuthorDate: Thu May 30 12:14:58 2024 +0800
[fix](inverted index)Change index_id from int32 to int64 to avoid overflow
#35206 (#35573)
Co-authored-by: qiye <[email protected]>
---
be/src/index-tools/index_tool.cpp | 4 ++--
be/src/olap/rowset/rowset_writer_context.h | 1 +
be/src/olap/rowset/segment_v2/inverted_index_desc.cpp | 6 +++---
be/src/olap/rowset/segment_v2/inverted_index_desc.h | 6 +++---
be/src/olap/rowset/segment_v2/inverted_index_reader.h | 2 +-
5 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/be/src/index-tools/index_tool.cpp
b/be/src/index-tools/index_tool.cpp
index 457560fdf05..9892d9d5bcb 100644
--- a/be/src/index-tools/index_tool.cpp
+++ b/be/src/index-tools/index_tool.cpp
@@ -51,7 +51,7 @@ DEFINE_string(pred_type, "", "inverted index term query
predicate, eq/lt/gt/le/g
DEFINE_bool(print_row_id, false, "print row id when query terms");
DEFINE_bool(print_doc_id, false, "print doc id when check terms stats");
// only for debug index compaction
-DEFINE_int32(idx_id, -1, "inverted index id");
+DEFINE_int64(idx_id, -1, "inverted index id");
DEFINE_string(src_idx_dirs_file, "", "source segment index files");
DEFINE_string(dest_idx_dirs_file, "", "destination segment index files");
DEFINE_string(dest_seg_num_rows_file, "", "destination segment number of
rows");
@@ -276,7 +276,7 @@ int main(int argc, char** argv) {
return true;
};
- int32_t index_id = FLAGS_idx_id;
+ int64_t index_id = FLAGS_idx_id;
std::string tablet_path = FLAGS_tablet_path;
std::string src_index_dirs_string;
std::string dest_index_dirs_string;
diff --git a/be/src/olap/rowset/rowset_writer_context.h
b/be/src/olap/rowset/rowset_writer_context.h
index f77108ee413..d2fa5998f2f 100644
--- a/be/src/olap/rowset/rowset_writer_context.h
+++ b/be/src/olap/rowset/rowset_writer_context.h
@@ -83,6 +83,7 @@ struct RowsetWriterContext {
int64_t newest_write_timestamp;
bool enable_unique_key_merge_on_write = false;
+ // store column_unique_id to skip write inverted index
std::set<int32_t> skip_inverted_index;
DataWriteType write_type = DataWriteType::TYPE_DEFAULT;
std::shared_ptr<Tablet> tablet = nullptr;
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
index 4bdf44ead06..ec5efd2553c 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
@@ -28,13 +28,13 @@ const std::string index_suffix = ".idx";
const std::string index_name_separator = "_";
std::string InvertedIndexDescriptor::get_temporary_index_path(const
std::string& segment_path,
- uint32_t uuid) {
+ uint64_t uuid) {
return StripSuffixString(segment_path, segment_suffix) +
index_name_separator +
std::to_string(uuid);
}
std::string InvertedIndexDescriptor::get_index_file_name(const std::string&
segment_path,
- uint32_t uuid) {
+ uint64_t uuid) {
return StripSuffixString(segment_path, segment_suffix) +
index_name_separator +
std::to_string(uuid) + index_suffix;
}
@@ -53,4 +53,4 @@ std::string
InvertedIndexDescriptor::local_inverted_index_path_segcompacted(
return fmt::format("{}/{}_{}-{}_{}.idx", tablet_path,
rowset_id.to_string(), begin, end,
index_id);
}
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.h
b/be/src/olap/rowset/segment_v2/inverted_index_desc.h
index 1cf4636d80b..dad45a3aa47 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_desc.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.h
@@ -28,8 +28,8 @@ namespace segment_v2 {
class InvertedIndexDescriptor {
public:
- static std::string get_temporary_index_path(const std::string&
segment_path, uint32_t uuid);
- static std::string get_index_file_name(const std::string& path, uint32_t
uuid);
+ static std::string get_temporary_index_path(const std::string&
segment_path, uint64_t uuid);
+ static std::string get_index_file_name(const std::string& path, uint64_t
uuid);
static const std::string get_temporary_null_bitmap_file_name() { return
"null_bitmap"; }
static const std::string get_temporary_bkd_index_data_file_name() { return
"bkd"; }
static const std::string get_temporary_bkd_index_meta_file_name() { return
"bkd_meta"; }
@@ -45,4 +45,4 @@ public:
};
} // namespace segment_v2
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
index 9c8d9dabf79..905fd12dfb3 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
@@ -91,7 +91,7 @@ public:
virtual InvertedIndexReaderType type() = 0;
bool indexExists(io::Path& index_file_path);
- [[nodiscard]] uint32_t get_index_id() const { return
_index_meta.index_id(); }
+ [[nodiscard]] uint64_t get_index_id() const { return
_index_meta.index_id(); }
[[nodiscard]] const std::map<string, string>& get_index_properties() const
{
return _index_meta.properties();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
