This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8391807af1b [improvement](inverted index)Avoid parsing ignore_above in
add_values and return error status when rowid_conversion is failed. (#32890)
8391807af1b is described below
commit 8391807af1bfe4bc6197d67034adcf8934bbf003
Author: qiye <[email protected]>
AuthorDate: Thu Mar 28 18:39:33 2024 +0800
[improvement](inverted index)Avoid parsing ignore_above in add_values and
return error status when rowid_conversion is failed. (#32890)
Co-authored-by: Luennng <[email protected]>
---
be/src/olap/compaction.cpp | 1 +
be/src/olap/rowset/segment_v2/inverted_index_writer.cpp | 14 ++++++--------
2 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 2947a7a5fc2..758c403cc2f 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -466,6 +466,7 @@ Status CompactionMixin::do_inverted_index_compaction() {
_tablet->table_id());
DCHECK(false) << err_msg;
LOG(WARNING) << err_msg;
+ return Status::InternalError(err_msg);
}
}
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index 87827464f15..c8cec0480e0 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -248,6 +248,9 @@ public:
// array's inverted index do need create field first
_doc->setNeedResetFieldData(true);
}
+ auto ignore_above_value =
+
get_parser_ignore_above_value_from_properties(_index_meta->properties());
+ _ignore_above = std::stoi(ignore_above_value);
return Status::OK();
}
@@ -324,13 +327,10 @@ public:
"field or index writer is null in inverted index
writer");
}
auto* v = (Slice*)values;
- auto ignore_above_value =
-
get_parser_ignore_above_value_from_properties(_index_meta->properties());
- auto ignore_above = std::stoi(ignore_above_value);
for (int i = 0; i < count; ++i) {
// only ignore_above UNTOKENIZED strings and empty strings not
tokenized
if ((_parser_type == InvertedIndexParserType::PARSER_NONE &&
- v->get_size() > ignore_above) ||
+ v->get_size() > _ignore_above) ||
(_parser_type != InvertedIndexParserType::PARSER_NONE &&
v->empty())) {
RETURN_IF_ERROR(add_null_document());
} else {
@@ -358,9 +358,6 @@ public:
LOG(ERROR) << "index writer is null in inverted index writer.";
return Status::InternalError("index writer is null in inverted
index writer");
}
- auto ignore_above_value =
-
get_parser_ignore_above_value_from_properties(_index_meta->properties());
- auto ignore_above = std::stoi(ignore_above_value);
for (int i = 0; i < count; ++i) {
// offsets[i+1] is now row element count
// [0, 3, 6]
@@ -383,7 +380,7 @@ public:
}
auto* v = (Slice*)((const uint8_t*)value_ptr + j *
field_size);
if ((_parser_type == InvertedIndexParserType::PARSER_NONE
&&
- v->get_size() > ignore_above) ||
+ v->get_size() > _ignore_above) ||
(_parser_type != InvertedIndexParserType::PARSER_NONE
&& v->empty())) {
// is here a null value?
// TODO. Maybe here has performance problem for large
size string.
@@ -615,6 +612,7 @@ private:
InvertedIndexParserType _parser_type;
std::wstring _field_name;
InvertedIndexFileWriter* _index_file_writer;
+ uint32_t _ignore_above;
};
Status InvertedIndexColumnWriter::create(const Field* field,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
