This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1ac5b451800cdb17906a8de498e5bccdee993d48 Author: zzzxl <[email protected]> AuthorDate: Thu Feb 1 18:01:52 2024 +0800 [fix](invert index) fixed the issue of insufficient index idx generation during partial column updates. (#30678) --- be/src/exec/tablet_info.cpp | 11 ++-- be/src/vec/functions/match.cpp | 21 +++++++ .../test_index_mow_fault_injection.out | 13 ++++ .../test_index_mow_fault_injection.groovy | 72 ++++++++++++++++++++++ 4 files changed, 112 insertions(+), 5 deletions(-) diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp index 5d89385a119..facb43ea403 100644 --- a/be/src/exec/tablet_info.cpp +++ b/be/src/exec/tablet_info.cpp @@ -201,23 +201,24 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) { } for (auto& t_index : tschema.indexes) { - std::unordered_map<std::string, SlotDescriptor*> index_slots_map; + std::unordered_map<std::string, int32_t> index_slots_map; auto index = _obj_pool.add(new OlapTableIndexSchema()); index->index_id = t_index.id; index->schema_hash = t_index.schema_hash; for (auto& tcolumn_desc : t_index.columns_desc) { - auto it = slots_map.find(std::make_pair(to_lower(tcolumn_desc.column_name), - thrift_to_type(tcolumn_desc.column_type.type))); if (!_is_partial_update || _partial_update_input_columns.count(tcolumn_desc.column_name) > 0) { + auto it = slots_map.find( + std::make_pair(to_lower(tcolumn_desc.column_name), + thrift_to_type(tcolumn_desc.column_type.type))); if (it == slots_map.end()) { return Status::InternalError("unknown index column, column={}, type={}", tcolumn_desc.column_name, tcolumn_desc.column_type.type); } - index_slots_map.emplace(to_lower(tcolumn_desc.column_name), it->second); index->slots.emplace_back(it->second); } + index_slots_map.emplace(to_lower(tcolumn_desc.column_name), tcolumn_desc.col_unique_id); TabletColumn* tc = _obj_pool.add(new TabletColumn()); tc->init_from_thrift(tcolumn_desc); index->columns.emplace_back(tc); @@ -228,7 +229,7 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) { for (size_t i = 0; i < tindex_desc.columns.size(); i++) { auto it = index_slots_map.find(to_lower(tindex_desc.columns[i])); if (it != index_slots_map.end()) { - column_unique_ids[i] = it->second->col_unique_id(); + column_unique_ids[i] = it->second; } } TabletIndex* ti = _obj_pool.add(new TabletIndex()); diff --git a/be/src/vec/functions/match.cpp b/be/src/vec/functions/match.cpp index 35fdb7a42b3..5002ef3f715 100644 --- a/be/src/vec/functions/match.cpp +++ b/be/src/vec/functions/match.cpp @@ -19,6 +19,7 @@ #include "runtime/query_context.h" #include "runtime/runtime_state.h" +#include "util/debug_points.h" namespace doris::vectorized { @@ -154,6 +155,11 @@ Status FunctionMatchAny::execute_match(const std::string& column_name, InvertedIndexCtx* inverted_index_ctx, const ColumnArray::Offsets64* array_offsets, ColumnUInt8::Container& result) const { + DBUG_EXECUTE_IF("match.invert_index_not_support_execute_match", { + return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>( + "FunctionMatchAny not support execute_match"); + }) + doris::InvertedIndexParserType parser_type = doris::InvertedIndexParserType::PARSER_UNKNOWN; if (inverted_index_ctx) { parser_type = inverted_index_ctx->parser_type; @@ -199,6 +205,11 @@ Status FunctionMatchAll::execute_match(const std::string& column_name, InvertedIndexCtx* inverted_index_ctx, const ColumnArray::Offsets64* array_offsets, ColumnUInt8::Container& result) const { + DBUG_EXECUTE_IF("match.invert_index_not_support_execute_match", { + return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>( + "FunctionMatchAll not support execute_match"); + }) + doris::InvertedIndexParserType parser_type = doris::InvertedIndexParserType::PARSER_UNKNOWN; if (inverted_index_ctx) { parser_type = inverted_index_ctx->parser_type; @@ -250,6 +261,11 @@ Status FunctionMatchPhrase::execute_match(const std::string& column_name, InvertedIndexCtx* inverted_index_ctx, const ColumnArray::Offsets64* array_offsets, ColumnUInt8::Container& result) const { + DBUG_EXECUTE_IF("match.invert_index_not_support_execute_match", { + return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>( + "FunctionMatchPhrase not support execute_match"); + }) + doris::InvertedIndexParserType parser_type = doris::InvertedIndexParserType::PARSER_UNKNOWN; if (inverted_index_ctx) { parser_type = inverted_index_ctx->parser_type; @@ -315,6 +331,11 @@ Status FunctionMatchPhrasePrefix::execute_match( const std::string& column_name, const std::string& match_query_str, size_t input_rows_count, const ColumnString* string_col, InvertedIndexCtx* inverted_index_ctx, const ColumnArray::Offsets64* array_offsets, ColumnUInt8::Container& result) const { + DBUG_EXECUTE_IF("match.invert_index_not_support_execute_match", { + return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>( + "FunctionMatchPhrasePrefix not support execute_match"); + }) + doris::InvertedIndexParserType parser_type = doris::InvertedIndexParserType::PARSER_UNKNOWN; if (inverted_index_ctx) { parser_type = inverted_index_ctx->parser_type; diff --git a/regression-test/data/fault_injection_p0/test_index_mow_fault_injection.out b/regression-test/data/fault_injection_p0/test_index_mow_fault_injection.out new file mode 100644 index 00000000000..e755b8cf9d8 --- /dev/null +++ b/regression-test/data/fault_injection_p0/test_index_mow_fault_injection.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +3 + +-- !sql -- +3 + +-- !sql -- +3 + +-- !sql -- +3 + diff --git a/regression-test/suites/fault_injection_p0/test_index_mow_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_index_mow_fault_injection.groovy new file mode 100644 index 00000000000..43c0c781163 --- /dev/null +++ b/regression-test/suites/fault_injection_p0/test_index_mow_fault_injection.groovy @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_index_mow_fault_injection", "inverted_index") { + // define a sql table + def testTable_unique = "httplogs_unique" + + def create_httplogs_unique_table = {testTablex -> + // multi-line sql + def result = sql """ + CREATE TABLE ${testTablex} ( + `@timestamp` int(11) NULL COMMENT "", + `clientip` string NULL COMMENT "", + `request` string NULL COMMENT "", + `status` string NULL COMMENT "", + `size` string NULL COMMENT "", + INDEX clientip_idx (`clientip`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true") COMMENT '', + INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true") COMMENT '', + INDEX status_idx (`status`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true") COMMENT '', + INDEX size_idx (`size`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true") COMMENT '' + ) ENGINE=OLAP + UNIQUE KEY(`@timestamp`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + } + + try { + sql "DROP TABLE IF EXISTS ${testTable_unique}" + create_httplogs_unique_table.call(testTable_unique) + + sql """ INSERT INTO ${testTable_unique} VALUES (893964617, '40.135.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 24736); """ + sql """ INSERT INTO ${testTable_unique} VALUES (893964653, '232.0.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 3781); """ + sql """ INSERT INTO ${testTable_unique} VALUES (893964672, '26.1.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 304, 0); """ + + sql """ update ${testTable_unique} set size = 1 where status = 200; """ + + sql 'sync' + + try { + GetDebugPoint().enableDebugPointForAllBEs("match.invert_index_not_support_execute_match") + + qt_sql """ select count() from ${testTable_unique} where (request match_phrase 'http'); """ + qt_sql """ select count() from ${testTable_unique} where (request match_phrase_prefix 'http'); """ + + qt_sql """ select count() from ${testTable_unique} where (clientip match_phrase 'http' or request match_phrase 'http' or status match_phrase 'http' or size match_phrase 'http'); """ + qt_sql """ select count() from ${testTable_unique} where (clientip match_phrase_prefix 'http' or request match_phrase_prefix 'http' or status match_phrase_prefix 'http' or size match_phrase_prefix 'http'); """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("match.invert_index_not_support_execute_match") + } + } finally { + } +} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
