This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new dff141fc40b [cherry-pick] (branch-2.0) fix query errors caused by
ignore_above (#37686)
dff141fc40b is described below
commit dff141fc40b7dcf866b914040a2f789c53d7e79b
Author: Sun Chenyang <[email protected]>
AuthorDate: Fri Jul 12 09:34:44 2024 +0800
[cherry-pick] (branch-2.0) fix query errors caused by ignore_above (#37686)
## Proposed changes
pick from master #37679
---
.../rowset/segment_v2/inverted_index_reader.cpp | 12 ++++++-
.../test_ignore_above_in_index.out | 4 +++
.../test_ignore_above_in_index.groovy | 42 ++++++++++++++++++++++
3 files changed, 57 insertions(+), 1 deletion(-)
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index b6b8f9c0441..f944ad7a510 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -629,8 +629,18 @@ Status
StringTypeInvertedIndexReader::query(OlapReaderStatistics* stats,
const StringRef* search_query = reinterpret_cast<const
StringRef*>(query_value);
auto act_len = strnlen(search_query->data, search_query->size);
+
+ // If the written value exceeds ignore_above, it will be written as null.
+ // The queried value exceeds ignore_above means the written value cannot
be found.
+ // The query needs to be downgraded to read from the segment file.
+ if (int ignore_above =
+
std::stoi(get_parser_ignore_above_value_from_properties(_index_meta.properties()));
+ act_len > ignore_above) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_EVALUATE_SKIPPED>(
+ "query value is too long, evaluate skipped.");
+ }
+
std::string search_str(search_query->data, act_len);
- // std::string search_str = reinterpret_cast<const
StringRef*>(query_value)->to_string();
VLOG_DEBUG << "begin to query the inverted index from clucene"
<< ", column_name: " << column_name << ", search_str: " <<
search_str;
std::wstring column_name_ws = StringUtil::string_to_wstring(column_name);
diff --git
a/regression-test/data/inverted_index_p0/test_ignore_above_in_index.out
b/regression-test/data/inverted_index_p0/test_ignore_above_in_index.out
new file mode 100644
index 00000000000..f88a155567e
--- /dev/null
+++ b/regression-test/data/inverted_index_p0/test_ignore_above_in_index.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+3
+
diff --git
a/regression-test/suites/inverted_index_p0/test_ignore_above_in_index.groovy
b/regression-test/suites/inverted_index_p0/test_ignore_above_in_index.groovy
new file mode 100644
index 00000000000..de508d9d263
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/test_ignore_above_in_index.groovy
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_ignore_above_in_index", "p0") {
+ def tableName = "test_ignore_above_in_index"
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName}(
+ `id`int(11)NULL,
+ `c` text NULL,
+ INDEX c_idx(`c`) USING INVERTED PROPERTIES("ignore_above"="9")
COMMENT ''
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES(
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ // ignore_above = 9, insert string length = 10
+ sql "insert into ${tableName} values (20, '1234567890');"
+ sql "insert into ${tableName} values (20, '1234567890');"
+ sql "insert into ${tableName} values (20, '1234567890');"
+ qt_sql "select count() from ${tableName} where c = '1234567890';"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
