This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new f77dd31cad4 [fix](invert index) fixed the issue of insufficient index
idx generation during partial column updates #30678 (#30640)
f77dd31cad4 is described below
commit f77dd31cad47b792b5067290010b396e96793466
Author: zzzxl <[email protected]>
AuthorDate: Sun Feb 4 20:39:27 2024 +0800
[fix](invert index) fixed the issue of insufficient index idx generation
during partial column updates #30678 (#30640)
---
be/src/exec/tablet_info.cpp | 14 ++---
be/src/vec/functions/match.cpp | 16 +++++
.../test_index_mow_fault_injection.out} | 11 +++-
.../test_index_mow_fault_injection.groovy | 72 ++++++++++++++++++++++
4 files changed, 105 insertions(+), 8 deletions(-)
diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp
index 6c3fed8f059..ac99ea63abb 100644
--- a/be/src/exec/tablet_info.cpp
+++ b/be/src/exec/tablet_info.cpp
@@ -150,25 +150,25 @@ Status OlapTableSchemaParam::init(const
TOlapTableSchemaParam& tschema) {
}
for (auto& t_index : tschema.indexes) {
- std::unordered_map<std::string, SlotDescriptor*> index_slots_map;
+ std::unordered_map<std::string, int32_t> index_slots_map;
auto index = _obj_pool.add(new OlapTableIndexSchema());
index->index_id = t_index.id;
index->schema_hash = t_index.schema_hash;
for (auto& tcolumn_desc : t_index.columns_desc) {
- TPrimitiveType::type col_type =
- has_invalid_type ? TPrimitiveType::INVALID_TYPE :
tcolumn_desc.column_type.type;
- auto it = slots_map.find(
- std::make_pair(to_lower(tcolumn_desc.column_name),
thrift_to_type(col_type)));
if (!_is_partial_update ||
_partial_update_input_columns.count(tcolumn_desc.column_name)
> 0) {
+ TPrimitiveType::type col_type = has_invalid_type ?
TPrimitiveType::INVALID_TYPE
+ :
tcolumn_desc.column_type.type;
+ auto it =
slots_map.find(std::make_pair(to_lower(tcolumn_desc.column_name),
+
thrift_to_type(col_type)));
if (it == slots_map.end()) {
return Status::InternalError("unknown index column,
column={}, type={}",
tcolumn_desc.column_name,
tcolumn_desc.column_type.type);
}
- index_slots_map.emplace(to_lower(tcolumn_desc.column_name),
it->second);
index->slots.emplace_back(it->second);
}
+ index_slots_map.emplace(to_lower(tcolumn_desc.column_name),
tcolumn_desc.col_unique_id);
TabletColumn* tc = _obj_pool.add(new TabletColumn());
tc->init_from_thrift(tcolumn_desc);
index->columns.emplace_back(tc);
@@ -179,7 +179,7 @@ Status OlapTableSchemaParam::init(const
TOlapTableSchemaParam& tschema) {
for (size_t i = 0; i < tindex_desc.columns.size(); i++) {
auto it =
index_slots_map.find(to_lower(tindex_desc.columns[i]));
if (it != index_slots_map.end()) {
- column_unique_ids[i] = it->second->col_unique_id();
+ column_unique_ids[i] = it->second;
}
}
TabletIndex* ti = _obj_pool.add(new TabletIndex());
diff --git a/be/src/vec/functions/match.cpp b/be/src/vec/functions/match.cpp
index c81c1617ca6..fa4301306ee 100644
--- a/be/src/vec/functions/match.cpp
+++ b/be/src/vec/functions/match.cpp
@@ -19,6 +19,7 @@
#include "runtime/query_context.h"
#include "runtime/runtime_state.h"
+#include "util/debug_points.h"
namespace doris::vectorized {
@@ -153,6 +154,11 @@ Status FunctionMatchAny::execute_match(const std::string&
column_name,
InvertedIndexCtx* inverted_index_ctx,
const ColumnArray::Offsets64*
array_offsets,
ColumnUInt8::Container& result) {
+ DBUG_EXECUTE_IF("match.invert_index_not_support_execute_match", {
+ return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
+ "FunctionMatchAny not support execute_match");
+ })
+
doris::InvertedIndexParserType parser_type =
doris::InvertedIndexParserType::PARSER_UNKNOWN;
if (inverted_index_ctx) {
parser_type = inverted_index_ctx->parser_type;
@@ -198,6 +204,11 @@ Status FunctionMatchAll::execute_match(const std::string&
column_name,
InvertedIndexCtx* inverted_index_ctx,
const ColumnArray::Offsets64*
array_offsets,
ColumnUInt8::Container& result) {
+ DBUG_EXECUTE_IF("match.invert_index_not_support_execute_match", {
+ return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
+ "FunctionMatchAll not support execute_match");
+ })
+
doris::InvertedIndexParserType parser_type =
doris::InvertedIndexParserType::PARSER_UNKNOWN;
if (inverted_index_ctx) {
parser_type = inverted_index_ctx->parser_type;
@@ -249,6 +260,11 @@ Status FunctionMatchPhrase::execute_match(const
std::string& column_name,
InvertedIndexCtx* inverted_index_ctx,
const ColumnArray::Offsets64*
array_offsets,
ColumnUInt8::Container& result) {
+ DBUG_EXECUTE_IF("match.invert_index_not_support_execute_match", {
+ return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
+ "FunctionMatchPhrase not support execute_match");
+ })
+
doris::InvertedIndexParserType parser_type =
doris::InvertedIndexParserType::PARSER_UNKNOWN;
if (inverted_index_ctx) {
parser_type = inverted_index_ctx->parser_type;
diff --git
a/regression-test/suites/fault_injection_p0/test_index_fault_injection.out
b/regression-test/data/fault_injection_p0/test_index_mow_fault_injection.out
similarity index 70%
rename from
regression-test/suites/fault_injection_p0/test_index_fault_injection.out
rename to
regression-test/data/fault_injection_p0/test_index_mow_fault_injection.out
index a21f3b8748e..e755b8cf9d8 100644
--- a/regression-test/suites/fault_injection_p0/test_index_fault_injection.out
+++ b/regression-test/data/fault_injection_p0/test_index_mow_fault_injection.out
@@ -1,4 +1,13 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !sql --
-0
+3
+
+-- !sql --
+3
+
+-- !sql --
+3
+
+-- !sql --
+3
diff --git
a/regression-test/suites/fault_injection_p0/test_index_mow_fault_injection.groovy
b/regression-test/suites/fault_injection_p0/test_index_mow_fault_injection.groovy
new file mode 100644
index 00000000000..43c0c781163
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/test_index_mow_fault_injection.groovy
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_index_mow_fault_injection", "inverted_index") {
+ // define a sql table
+ def testTable_unique = "httplogs_unique"
+
+ def create_httplogs_unique_table = {testTablex ->
+ // multi-line sql
+ def result = sql """
+ CREATE TABLE ${testTablex} (
+ `@timestamp` int(11) NULL COMMENT "",
+ `clientip` string NULL COMMENT "",
+ `request` string NULL COMMENT "",
+ `status` string NULL COMMENT "",
+ `size` string NULL COMMENT "",
+ INDEX clientip_idx (`clientip`) USING INVERTED PROPERTIES("parser" =
"unicode", "support_phrase" = "true") COMMENT '',
+ INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" =
"unicode", "support_phrase" = "true") COMMENT '',
+ INDEX status_idx (`status`) USING INVERTED PROPERTIES("parser" =
"unicode", "support_phrase" = "true") COMMENT '',
+ INDEX size_idx (`size`) USING INVERTED PROPERTIES("parser" =
"unicode", "support_phrase" = "true") COMMENT ''
+ ) ENGINE=OLAP
+ UNIQUE KEY(`@timestamp`)
+ COMMENT "OLAP"
+ DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "enable_unique_key_merge_on_write" = "true"
+ );
+ """
+ }
+
+ try {
+ sql "DROP TABLE IF EXISTS ${testTable_unique}"
+ create_httplogs_unique_table.call(testTable_unique)
+
+ sql """ INSERT INTO ${testTable_unique} VALUES (893964617, '40.135.0.0',
'GET /images/hm_bg.jpg HTTP/1.0', 200, 24736); """
+ sql """ INSERT INTO ${testTable_unique} VALUES (893964653, '232.0.0.0',
'GET /images/hm_bg.jpg HTTP/1.0', 200, 3781); """
+ sql """ INSERT INTO ${testTable_unique} VALUES (893964672, '26.1.0.0',
'GET /images/hm_bg.jpg HTTP/1.0', 304, 0); """
+
+ sql """ update ${testTable_unique} set size = 1 where status = 200; """
+
+ sql 'sync'
+
+ try {
+
GetDebugPoint().enableDebugPointForAllBEs("match.invert_index_not_support_execute_match")
+
+ qt_sql """ select count() from ${testTable_unique} where (request
match_phrase 'http'); """
+ qt_sql """ select count() from ${testTable_unique} where (request
match_phrase_prefix 'http'); """
+
+ qt_sql """ select count() from ${testTable_unique} where (clientip
match_phrase 'http' or request match_phrase 'http' or status match_phrase
'http' or size match_phrase 'http'); """
+ qt_sql """ select count() from ${testTable_unique} where (clientip
match_phrase_prefix 'http' or request match_phrase_prefix 'http' or status
match_phrase_prefix 'http' or size match_phrase_prefix 'http'); """
+ } finally {
+
GetDebugPoint().disableDebugPointForAllBEs("match.invert_index_not_support_execute_match")
+ }
+ } finally {
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
