This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b839353c2d [fix](inverted index) fix BE coredump because of not ignore
case ensitivity for column name when create index (#17276)
b839353c2d is described below
commit b839353c2dd2f8f72b15eea82e4a86f486ba58ae
Author: YueW <[email protected]>
AuthorDate: Wed Mar 1 19:32:39 2023 +0800
[fix](inverted index) fix BE coredump because of not ignore case ensitivity
for column name when create index (#17276)
---
be/src/olap/schema_change.cpp | 30 +++-
be/src/olap/tablet_schema.cpp | 7 +-
.../apache/doris/alter/SchemaChangeHandler.java | 10 +-
.../java/org/apache/doris/analysis/IndexDef.java | 8 +
.../test_add_drop_index_ignore_case_column.groovy | 194 +++++++++++++++++++++
5 files changed, 243 insertions(+), 6 deletions(-)
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index af254038bc..1735c3debc 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -627,8 +627,14 @@ Status
SchemaChangeForInvertedIndex::process(RowsetReaderSharedPtr rowset_reader
DCHECK_EQ(inverted_index.columns.size(), 1);
auto index_id = inverted_index.index_id;
auto column_name = inverted_index.columns[0];
- auto column = _tablet_schema->column(column_name);
auto column_idx = _tablet_schema->field_index(column_name);
+ if (column_idx < 0) {
+ LOG(WARNING) << "referenced column was missing. "
+ << "[column=" << column_name << "
referenced_column=" << column_idx
+ << "]";
+ return Status::Error<CE_CMD_PARAMS_ERROR>();
+ }
+ auto column = _tablet_schema->column(column_idx);
return_columns.emplace_back(column_idx);
_olap_data_convertor->add_column_data_convertor(column);
@@ -782,7 +788,13 @@ Status
SchemaChangeForInvertedIndex::_write_inverted_index(int32_t segment_idx,
int idx = 0;
for (auto& inverted_index : _alter_inverted_indexs) {
auto column_name = inverted_index.columns[0];
- auto column = _tablet_schema->column(column_name);
+ auto column_idx = _tablet_schema->field_index(column_name);
+ if (column_idx < 0) {
+ LOG(WARNING) << "referenced column was missing. "
+ << "[column=" << column_name << " referenced_column="
<< column_idx << "]";
+ return Status::Error<CE_CMD_PARAMS_ERROR>();
+ }
+ auto column = _tablet_schema->column(column_idx);
auto index_id = inverted_index.index_id;
auto converted_result =
_olap_data_convertor->convert_column_data(idx++);
@@ -1275,6 +1287,11 @@ Status
SchemaChangeHandler::_get_rowset_readers(TabletSharedPtr tablet,
DCHECK_EQ(inverted_index.columns.size(), 1);
auto column_name = inverted_index.columns[0];
auto idx = tablet_schema->field_index(column_name);
+ if (idx < 0) {
+ LOG(WARNING) << "referenced column was missing. "
+ << "[column=" << column_name << " referenced_column="
<< idx << "]";
+ return Status::Error<CE_CMD_PARAMS_ERROR>();
+ }
return_columns.emplace_back(idx);
}
@@ -1382,7 +1399,14 @@ Status
SchemaChangeHandler::_drop_inverted_index(std::vector<RowsetReaderSharedP
rowset_meta->rowset_id(), i);
for (auto& inverted_index : alter_inverted_indexs) {
auto column_name = inverted_index.columns[0];
- auto column = tablet_schema->column(column_name);
+ auto column_idx = tablet_schema->field_index(column_name);
+ if (column_idx < 0) {
+ LOG(WARNING) << "referenced column was missing. "
+ << "[column=" << column_name << "
referenced_column=" << column_idx
+ << "]";
+ return Status::Error<CE_CMD_PARAMS_ERROR>();
+ }
+ auto column = tablet_schema->column(column_idx);
auto index_id = inverted_index.index_id;
std::string inverted_index_file =
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 1c77ac3234..dcfac6ef7e 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -503,7 +503,12 @@ void TabletIndex::init_from_thrift(const TOlapTableIndex&
index,
// get column unique id by name
std::vector<int32_t> col_unique_ids(index.columns.size());
for (size_t i = 0; i < index.columns.size(); i++) {
- col_unique_ids[i] = tablet_schema.column(index.columns[i]).unique_id();
+ auto column_idx = tablet_schema.field_index(index.columns[i]);
+ if (column_idx >= 0) {
+ col_unique_ids[i] = tablet_schema.column(column_idx).unique_id();
+ } else {
+ col_unique_ids[i] = -1;
+ }
}
_col_unique_ids = std::move(col_unique_ids);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 3bfee997fd..11fb77c3ee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -2128,7 +2128,8 @@ public class SchemaChangeHandler extends AlterHandler {
*/
private boolean processAddIndex(CreateIndexClause alterClause, OlapTable
olapTable, List<Index> newIndexes)
throws UserException {
- if (alterClause.getIndex() == null) {
+ Index alterIndex = alterClause.getIndex();
+ if (alterIndex == null) {
return false;
}
@@ -2166,7 +2167,12 @@ public class SchemaChangeHandler extends AlterHandler {
}
}
- newIndexes.add(alterClause.getIndex());
+ // the column name in CreateIndexClause is not check case sensitivity,
+ // when send index description to BE, there maybe cannot find column
by name,
+ // so here update column name in CreateIndexClause after checkColumn
for indexDef,
+ // there will use the column name in olapTable insead of the column
name in CreateIndexClause.
+ alterIndex.setColumns(indexDef.getColumns());
+ newIndexes.add(alterIndex);
return false;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
index d1c21b5d37..711a83496b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.AnalysisException;
import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
import java.util.HashMap;
import java.util.List;
@@ -34,6 +35,9 @@ public class IndexDef {
private String indexName;
private boolean ifNotExists;
private List<String> columns;
+ // add the column name of olapTable column into caseSensitivityColumns
+ // instead of the column name which from sql_parser analyze
+ private List<String> caseSensitivityColumns = Lists.newArrayList();
private IndexType indexType;
private String comment;
private Map<String, String> properties;
@@ -142,6 +146,9 @@ public class IndexDef {
}
public List<String> getColumns() {
+ if (caseSensitivityColumns.size() > 0) {
+ return caseSensitivityColumns;
+ }
return columns;
}
@@ -176,6 +183,7 @@ public class IndexDef {
if (indexType == IndexType.BITMAP || indexType == IndexType.INVERTED
|| indexType == IndexType.BLOOMFILTER
|| indexType == IndexType.NGRAM_BF) {
String indexColName = column.getName();
+ caseSensitivityColumns.add(indexColName);
PrimitiveType colType = column.getDataType();
if (indexType == IndexType.INVERTED && colType.isArrayType()) {
colType = ((ArrayType)
column.getType()).getItemType().getPrimitiveType();
diff --git
a/regression-test/suites/inverted_index_p0/test_add_drop_index_ignore_case_column.groovy
b/regression-test/suites/inverted_index_p0/test_add_drop_index_ignore_case_column.groovy
new file mode 100644
index 0000000000..ca8e390f67
--- /dev/null
+++
b/regression-test/suites/inverted_index_p0/test_add_drop_index_ignore_case_column.groovy
@@ -0,0 +1,194 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_add_drop_index_with_ignore_case_column", "inverted_index"){
+ // prepare test table
+ def timeout = 60000
+ def delta_time = 1000
+ def alter_res = "null"
+ def useTime = 0
+ def wait_for_latest_op_on_table_finish = { table_name, OpTimeout ->
+ for(int t = delta_time; t <= OpTimeout; t += delta_time){
+ alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName =
"${table_name}" ORDER BY CreateTime DESC LIMIT 1;"""
+ alter_res = alter_res.toString()
+ if(alter_res.contains("FINISHED")) {
+ break
+ }
+ useTime = t
+ sleep(delta_time)
+ }
+ assertTrue(useTime <= OpTimeout)
+ }
+
+ def indexTbName1 = "test_add_drop_inverted_index4"
+
+ sql "DROP TABLE IF EXISTS ${indexTbName1}"
+ // create 1 replica table
+ sql """
+ CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+ `id` int(11) NULL,
+ `Name` text NULL,
+ `Description` text NULL,
+ INDEX idx_id (`id`) USING INVERTED COMMENT '',
+ INDEX idx_name (`name`) USING INVERTED
PROPERTIES("parser"="none") COMMENT ''
+ )
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 10
+ properties("replication_num" = "1");
+ """
+
+ // set enable_vectorized_engine=true
+ sql """ SET enable_vectorized_engine=true; """
+ def var_result = sql "show variables"
+ logger.info("show variales result: " + var_result )
+
+ // show index of create table
+ def show_result = sql "show index from ${indexTbName1}"
+ logger.info("show index from " + indexTbName1 + " result: " + show_result)
+ assertEquals(show_result[0][2], "idx_id")
+ assertEquals(show_result[1][2], "idx_name")
+
+ // insert data
+ sql "insert into ${indexTbName1} values (1, 'name1', 'desc 1'), (2,
'name2', 'desc 2')"
+
+ // query all rows
+ def select_result = sql "select * from ${indexTbName1} order by id"
+ assertEquals(select_result.size(), 2)
+ assertEquals(select_result[0][0], 1)
+ assertEquals(select_result[0][1], "name1")
+ assertEquals(select_result[0][2], "desc 1")
+ assertEquals(select_result[1][0], 2)
+ assertEquals(select_result[1][1], "name2")
+ assertEquals(select_result[1][2], "desc 2")
+
+ // query rows where name='name1'
+ select_result = sql "select * from ${indexTbName1} where name='name1'"
+ assertEquals(select_result.size(), 1)
+ assertEquals(select_result[0][0], 1)
+ assertEquals(select_result[0][1], "name1")
+ assertEquals(select_result[0][2], "desc 1")
+
+ // query rows where name match 'name2'
+ select_result = sql "select * from ${indexTbName1} where name match
'name2'"
+ assertEquals(select_result.size(), 1)
+ assertEquals(select_result[0][0], 2)
+ assertEquals(select_result[0][1], "name2")
+ assertEquals(select_result[0][2], "desc 2")
+
+ // query rows where description match 'desc', should fail without index
+ def success = false
+ try {
+ sql "select * from ${indexTbName1} where description match 'desc'"
+ success = true
+ } catch(Exception ex) {
+ logger.info("sql exception: " + ex)
+ }
+ assertEquals(success, false)
+
+ // add index on column description
+ sql "create index idx_desc on ${indexTbName1}(description) USING INVERTED
PROPERTIES(\"parser\"=\"standard\");"
+ wait_for_latest_op_on_table_finish(indexTbName1, timeout)
+
+ // show index after add index
+ show_result = sql "show index from ${indexTbName1}"
+ logger.info("show index from " + indexTbName1 + " result: " + show_result)
+ assertEquals(show_result[0][2], "idx_id")
+ assertEquals(show_result[1][2], "idx_name")
+ assertEquals(show_result[2][2], "idx_desc")
+
+ // query rows where description match 'desc'
+ select_result = sql "select * from ${indexTbName1} where description match
'desc' order by id"
+ assertEquals(select_result.size(), 2)
+ assertEquals(select_result[0][0], 1)
+ assertEquals(select_result[0][1], "name1")
+ assertEquals(select_result[0][2], "desc 1")
+ assertEquals(select_result[1][0], 2)
+ assertEquals(select_result[1][1], "name2")
+ assertEquals(select_result[1][2], "desc 2")
+
+ // query rows where description match_all 'desc 1'
+ select_result = sql "select * from ${indexTbName1} where description
match_all 'desc 1'"
+ assertEquals(select_result.size(), 1)
+ assertEquals(select_result[0][0], 1)
+ assertEquals(select_result[0][1], "name1")
+ assertEquals(select_result[0][2], "desc 1")
+
+ // query rows where description match_all 'desc 2'
+ select_result = sql "select * from ${indexTbName1} where description
match_all 'desc 2'"
+ assertEquals(select_result.size(), 1)
+ assertEquals(select_result[0][0], 2)
+ assertEquals(select_result[0][1], "name2")
+ assertEquals(select_result[0][2], "desc 2")
+
+ // drop index
+ // add index on column description
+ sql "drop index idx_desc on ${indexTbName1}"
+ wait_for_latest_op_on_table_finish(indexTbName1, timeout)
+
+ // query rows where description match 'desc', should fail without index
+ success = false
+ try {
+ sql "select * from ${indexTbName1} where description match 'desc'"
+ success = true
+ } catch(Exception ex) {
+ logger.info("sql exception: " + ex)
+ }
+ assertEquals(success, false)
+
+ // query rows where name='name1'
+ select_result = sql "select * from ${indexTbName1} where name='name1'"
+ assertEquals(select_result.size(), 1)
+ assertEquals(select_result[0][0], 1)
+ assertEquals(select_result[0][1], "name1")
+ assertEquals(select_result[0][2], "desc 1")
+
+ // query rows where name='name2'
+ select_result = sql "select * from ${indexTbName1} where name='name2'"
+ assertEquals(select_result.size(), 1)
+ assertEquals(select_result[0][0], 2)
+ assertEquals(select_result[0][1], "name2")
+ assertEquals(select_result[0][2], "desc 2")
+
+ // add index on column description
+ sql "create index idx_desc on ${indexTbName1}(DESCRIPTION) USING INVERTED
PROPERTIES(\"parser\"=\"standard\");"
+ wait_for_latest_op_on_table_finish(indexTbName1, timeout)
+
+ // query rows where description match 'desc'
+ select_result = sql "select * from ${indexTbName1} where description match
'desc' order by id"
+ assertEquals(select_result.size(), 2)
+ assertEquals(select_result[0][0], 1)
+ assertEquals(select_result[0][1], "name1")
+ assertEquals(select_result[0][2], "desc 1")
+ assertEquals(select_result[1][0], 2)
+ assertEquals(select_result[1][1], "name2")
+ assertEquals(select_result[1][2], "desc 2")
+
+ // query rows where description match_all 'desc 1'
+ select_result = sql "select * from ${indexTbName1} where description
match_all 'desc 1'"
+ assertEquals(select_result.size(), 1)
+ assertEquals(select_result[0][0], 1)
+ assertEquals(select_result[0][1], "name1")
+ assertEquals(select_result[0][2], "desc 1")
+
+ // query rows where description match_all 'desc 2'
+ select_result = sql "select * from ${indexTbName1} where description
match_all 'desc 2'"
+ assertEquals(select_result.size(), 1)
+ assertEquals(select_result[0][0], 2)
+ assertEquals(select_result[0][1], "name2")
+ assertEquals(select_result[0][2], "desc 2")
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
