This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new efe17245b09 [Fix](inverted index) fix index_id wrong size in V2
(#35909)
efe17245b09 is described below
commit efe17245b09391105974d9229c06c467217f7676
Author: airborne12 <[email protected]>
AuthorDate: Wed Jun 5 20:36:40 2024 +0800
[Fix](inverted index) fix index_id wrong size in V2 (#35909)
This pull request modifies the index_id type in inverted index storage
format v2 to int64_t. The index_id is now stored in the inverted index
file using 4 bytes.
---
be/src/olap/rowset/segment_v2/inverted_index_desc.cpp | 10 +++++-----
be/src/olap/rowset/segment_v2/inverted_index_desc.h | 6 +++---
be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp | 4 ++--
be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp | 6 +++---
4 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
index 1761f8f39d0..7fe23482d72 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
@@ -28,18 +28,18 @@ const std::string InvertedIndexDescriptor::index_suffix =
".idx";
const std::string InvertedIndexDescriptor::index_name_separator = "_";
std::string InvertedIndexDescriptor::get_temporary_index_path(
- const std::string& segment_path, uint64_t uuid, const std::string&
index_suffix_path) {
+ const std::string& segment_path, int64_t index_id, const std::string&
index_suffix_path) {
std::string suffix = index_suffix_path.empty() ? "" : "@" +
index_suffix_path;
return StripSuffixString(segment_path, segment_suffix) +
index_name_separator +
- std::to_string(uuid) + suffix;
+ std::to_string(index_id) + suffix;
}
std::string InvertedIndexDescriptor::get_index_file_name(const std::string&
segment_path,
- uint64_t uuid,
+ int64_t index_id,
const std::string&
index_suffix_path) {
std::string suffix = index_suffix_path.empty() ? "" : "@" +
index_suffix_path;
return StripSuffixString(segment_path, segment_suffix) +
index_name_separator +
- std::to_string(uuid) + suffix + index_suffix;
+ std::to_string(index_id) + suffix + index_suffix;
}
std::string InvertedIndexDescriptor::inverted_index_file_path(
@@ -63,4 +63,4 @@ std::string
InvertedIndexDescriptor::local_inverted_index_path_segcompacted(
return fmt::format("{}/{}_{}-{}_{}{}.idx", tablet_path,
rowset_id.to_string(), begin, end,
index_id, suffix);
}
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.h
b/be/src/olap/rowset/segment_v2/inverted_index_desc.h
index 0f9d591be6c..92bcf0e127c 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_desc.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.h
@@ -31,9 +31,9 @@ public:
static const std::string segment_suffix;
static const std::string index_suffix;
static const std::string index_name_separator;
- static std::string get_temporary_index_path(const std::string&
segment_path, uint64_t uuid,
+ static std::string get_temporary_index_path(const std::string&
segment_path, int64_t index_id,
const std::string&
index_suffix_path);
- static std::string get_index_file_name(const std::string& path, uint64_t
uuid,
+ static std::string get_index_file_name(const std::string& path, int64_t
index_id,
const std::string&
index_suffix_path);
static std::string get_index_file_name(const std::string& path);
static const std::string get_temporary_null_bitmap_file_name() { return
"null_bitmap"; }
@@ -53,4 +53,4 @@ public:
};
} // namespace segment_v2
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
index ccbb0eb4c0b..c21ece4b86c 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp
@@ -72,7 +72,7 @@ Status InvertedIndexFileReader::_init_from_v2(int32_t
read_buffer_size) {
ReaderFileEntry* entry = nullptr;
for (int32_t i = 0; i < numIndices; ++i) {
- int64_t indexId = _stream->readInt(); // Read index ID
+ int64_t index_id = _stream->readLong(); // Read index ID
int32_t suffix_length = _stream->readInt(); // Read suffix
length
std::vector<uint8_t> suffix_data(suffix_length);
_stream->readBytes(suffix_data.data(), suffix_length);
@@ -99,7 +99,7 @@ Status InvertedIndexFileReader::_init_from_v2(int32_t
read_buffer_size) {
fileEntries->put(aid, entry);
}
- _indices_entries.emplace(std::make_pair(indexId,
std::move(suffix_str)),
+ _indices_entries.emplace(std::make_pair(index_id,
std::move(suffix_str)),
std::move(fileEntries));
}
} else {
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
index 5727a452bf1..675c55bc77e 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
@@ -100,7 +100,7 @@ size_t InvertedIndexFileWriter::headerLength() {
sizeof(int) * 2; // Account for the size of the version number and
number of indices
for (const auto& entry : _indices_dirs) {
auto suffix = entry.first.second;
- header_size += sizeof(int); // index id
+ header_size += sizeof(int64_t); // index id
header_size += 4; // index suffix name size
header_size += suffix.length(); // index suffix name
header_size += sizeof(int); // index file count
@@ -199,7 +199,7 @@ size_t InvertedIndexFileWriter::write() {
int32_t file_count = sorted_files.size();
// Write the index ID and the number of files
- compound_file_output->writeInt(index_id);
+ compound_file_output->writeLong(index_id);
const auto* index_suffix_str = reinterpret_cast<const
uint8_t*>(index_suffix.c_str());
compound_file_output->writeInt(index_suffix.length());
compound_file_output->writeBytes(index_suffix_str,
index_suffix.length());
@@ -413,4 +413,4 @@ void DorisCompoundFileWriter::copyFile(const char*
fileName, lucene::store::Dire
}
input->close();
}
-} // namespace doris::segment_v2
\ No newline at end of file
+} // namespace doris::segment_v2
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
