This is an automated email from the ASF dual-hosted git repository.
cambyzju pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 746c6207fca [fix](index) bitmap and bloomfilter index should not do
light index change (#35225)
746c6207fca is described below
commit 746c6207fca8aef5f66c29b903d407995fdf1923
Author: camby <[email protected]>
AuthorDate: Wed May 29 10:09:31 2024 +0800
[fix](index) bitmap and bloomfilter index should not do light index change
(#35225)
---
.../olap/rowset/segment_v2/bitmap_index_reader.cpp | 3 +
.../segment_v2/bloom_filter_index_reader.cpp | 4 +
be/src/olap/schema_change.cpp | 12 +++
be/src/olap/tablet_schema.cpp | 18 ++++
be/src/olap/tablet_schema.h | 2 +
.../test_index_ddl_fault_injection.out | 22 +++++
.../test_index_ddl_fault_injection.groovy | 97 ++++++++++++++++++++++
7 files changed, 158 insertions(+)
diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp
b/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp
index c76de68b7ba..fe3609f01ad 100644
--- a/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp
@@ -24,6 +24,7 @@
#include <roaring/roaring.hh>
#include "olap/types.h"
+#include "util/debug_points.h"
#include "vec/columns/column.h"
#include "vec/common/string_ref.h"
#include "vec/data_types/data_type.h"
@@ -53,6 +54,8 @@ Status BitmapIndexReader::_load(bool use_page_cache, bool
kept_in_memory,
}
Status BitmapIndexReader::new_iterator(BitmapIndexIterator** iterator) {
+ DBUG_EXECUTE_IF("BitmapIndexReader::new_iterator.fail",
+ { return Status::InternalError("new_iterator for bitmap
index failed"); });
*iterator = new BitmapIndexIterator(this);
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
index c9b74c034ee..0857c1890c4 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
@@ -22,6 +22,7 @@
#include "olap/rowset/segment_v2/bloom_filter.h"
#include "olap/types.h"
+#include "util/debug_points.h"
#include "vec/columns/column.h"
#include "vec/common/string_ref.h"
#include "vec/data_types/data_type.h"
@@ -46,6 +47,9 @@ Status BloomFilterIndexReader::_load(bool use_page_cache,
bool kept_in_memory) {
}
Status
BloomFilterIndexReader::new_iterator(std::unique_ptr<BloomFilterIndexIterator>*
iterator) {
+ DBUG_EXECUTE_IF("BloomFilterIndexReader::new_iterator.fail", {
+ return Status::InternalError("new_iterator for bloom filter index
failed");
+ });
iterator->reset(new BloomFilterIndexIterator(this));
return Status::OK();
}
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index c5ae7793c47..02f60a993a1 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -765,6 +765,9 @@ Status
SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2&
for (const auto& column : request.columns) {
base_tablet_schema->append_column(TabletColumn(column));
}
+ // The request only include column info, do not include bitmap or
bloomfilter index info,
+ // So we also need to copy index info from the real base tablet
+
base_tablet_schema->update_index_info_from(*base_tablet->tablet_schema());
}
// Use tablet schema directly from base tablet, they are the newest
schema, not contain
// dropped column during light weight schema change.
@@ -1299,6 +1302,15 @@ Status SchemaChangeHandler::_parse_request(const
SchemaChangeParams& sc_params,
if (column_mapping->expr != nullptr) {
*sc_directly = true;
return Status::OK();
+ } else if (column_mapping->ref_column >= 0) {
+ const auto& column_new = new_tablet_schema->column(i);
+ const auto& column_old =
base_tablet_schema->column(column_mapping->ref_column);
+ // index changed
+ if (column_new.is_bf_column() != column_old.is_bf_column() ||
+ column_new.has_bitmap_index() !=
column_old.has_bitmap_index()) {
+ *sc_directly = true;
+ return Status::OK();
+ }
}
}
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index d423ec4d709..a44abedc354 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -974,6 +974,24 @@ void TabletSchema::copy_from(const TabletSchema&
tablet_schema) {
_table_id = tablet_schema.table_id();
}
+void TabletSchema::update_index_info_from(const TabletSchema& tablet_schema) {
+ for (auto& col : _cols) {
+ if (col->unique_id() < 0) {
+ continue;
+ }
+ const auto iter =
tablet_schema._field_id_to_index.find(col->unique_id());
+ if (iter == tablet_schema._field_id_to_index.end()) {
+ continue;
+ }
+ int32_t col_idx = iter->second;
+ if (col_idx < 0 || col_idx >= tablet_schema._cols.size()) {
+ continue;
+ }
+ col->set_is_bf_column(tablet_schema._cols[col_idx]->is_bf_column());
+
col->set_has_bitmap_index(tablet_schema._cols[col_idx]->has_bitmap_index());
+ }
+}
+
std::string TabletSchema::to_key() const {
TabletSchemaPB pb;
to_schema_pb(&pb);
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 384772375d1..c9e8fdce54e 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -162,6 +162,7 @@ public:
int32_t parent_unique_id() const { return _parent_col_unique_id; }
void set_parent_unique_id(int32_t col_unique_id) { _parent_col_unique_id =
col_unique_id; }
void set_is_bf_column(bool is_bf_column) { _is_bf_column = is_bf_column; }
+ void set_has_bitmap_index(bool has_bitmap_index) { _has_bitmap_index =
has_bitmap_index; }
std::shared_ptr<const vectorized::IDataType> get_vec_type() const;
void append_sparse_column(TabletColumn column);
@@ -282,6 +283,7 @@ public:
// Must make sure the row column is always the last column
void add_row_column();
void copy_from(const TabletSchema& tablet_schema);
+ void update_index_info_from(const TabletSchema& tablet_schema);
std::string to_key() const;
// Don't use.
// TODO: memory size of TabletSchema cannot be accurately tracked.
diff --git
a/regression-test/data/fault_injection_p0/test_index_ddl_fault_injection.out
b/regression-test/data/fault_injection_p0/test_index_ddl_fault_injection.out
new file mode 100644
index 00000000000..6bbc4f12702
--- /dev/null
+++ b/regression-test/data/fault_injection_p0/test_index_ddl_fault_injection.out
@@ -0,0 +1,22 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !order1 --
+1 2 hello
+
+-- !order2 --
+1 2 hello
+
+-- !order3 --
+1 2 hello
+
+-- !order4 --
+1 2 hello
+
+-- !order5 --
+1 2 hello
+
+-- !order6 --
+1 2 hello
+
+-- !order7 --
+1 2 hello
+
diff --git
a/regression-test/suites/fault_injection_p0/test_index_ddl_fault_injection.groovy
b/regression-test/suites/fault_injection_p0/test_index_ddl_fault_injection.groovy
new file mode 100644
index 00000000000..0beef4e80a6
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/test_index_ddl_fault_injection.groovy
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_index_ddl_fault_injection", "nonConcurrent") {
+ try {
+ sql "DROP TABLE IF EXISTS `test_index_ddl_fault_injection_tbl`"
+ sql """
+ CREATE TABLE test_index_ddl_fault_injection_tbl (
+ `k1` int(11) NULL COMMENT "",
+ `k2` int(11) NULL COMMENT "",
+ `v1` string NULL COMMENT ""
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`k1`)
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+ PROPERTIES ( "replication_allocation" = "tag.location.default: 1");
+ """
+
+ sql """ INSERT INTO test_index_ddl_fault_injection_tbl VALUES (1, 2,
"hello"), (3, 4, "world"); """
+ sql 'sync'
+
+ qt_order1 """ select * from test_index_ddl_fault_injection_tbl where v1
= 'hello'; """
+
+ // add bloom filter index
+ sql """ ALTER TABLE test_index_ddl_fault_injection_tbl set
("bloom_filter_columns" = "v1"); """
+ assertEquals("FINISHED",
getAlterColumnFinalState("test_index_ddl_fault_injection_tbl"))
+
+ try {
+ qt_order2 """ select * from test_index_ddl_fault_injection_tbl where
v1 = 'hello'; """
+
GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexReader::new_iterator.fail");
+ test {
+ // if BE add bloom filter correctly, this query will call
BloomFilterIndexReader::new_iterator
+ sql """ select * from test_index_ddl_fault_injection_tbl where v1
= 'hello'; """
+ exception "new_iterator for bloom filter index failed"
+ }
+ } finally {
+
GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexReader::new_iterator.fail");
+ }
+
+ // drop bloom filter index
+ sql """ ALTER TABLE test_index_ddl_fault_injection_tbl set
("bloom_filter_columns" = ""); """
+ assertEquals("FINISHED",
getAlterColumnFinalState("test_index_ddl_fault_injection_tbl"))
+
+ try {
+ qt_order3 """ select * from test_index_ddl_fault_injection_tbl where
v1 = 'hello'; """
+
GetDebugPoint().enableDebugPointForAllBEs("BloomFilterIndexReader::new_iterator.fail");
+ // if BE drop bloom filter correctly, this query will not call
BloomFilterIndexReader::new_iterator
+ qt_order4 """ select * from test_index_ddl_fault_injection_tbl where
v1 = 'hello'; """
+ } finally {
+
GetDebugPoint().disableDebugPointForAllBEs("BloomFilterIndexReader::new_iterator.fail");
+ }
+
+ // add bitmap index
+ sql """ ALTER TABLE test_index_ddl_fault_injection_tbl ADD INDEX
idx_bitmap(v1) USING BITMAP; """
+ assertEquals("FINISHED",
getAlterColumnFinalState("test_index_ddl_fault_injection_tbl"))
+ try {
+ qt_order5 """ select * from test_index_ddl_fault_injection_tbl where
v1 = 'hello'; """
+
GetDebugPoint().enableDebugPointForAllBEs("BitmapIndexReader::new_iterator.fail");
+ test {
+ // if BE add bitmap index correctly, this query will call
BitmapIndexReader::new_iterator
+ sql """ select * from test_index_ddl_fault_injection_tbl where v1
= 'hello'; """
+ exception "new_iterator for bitmap index failed"
+ }
+ } finally {
+
GetDebugPoint().disableDebugPointForAllBEs("BitmapIndexReader::new_iterator.fail");
+ }
+
+ // drop bitmap index
+ sql """ DROP INDEX idx_bitmap ON test_index_ddl_fault_injection_tbl; """
+ assertEquals("FINISHED",
getAlterColumnFinalState("test_index_ddl_fault_injection_tbl"))
+
+ try {
+ qt_order6 """ select * from test_index_ddl_fault_injection_tbl where
v1 = 'hello'; """
+
GetDebugPoint().enableDebugPointForAllBEs("BitmapIndexReader::new_iterator.fail");
+ // if BE drop bitmap index correctly, this query will not call
BitmapIndexReader::new_iterator
+ qt_order7 """ select * from test_index_ddl_fault_injection_tbl where
v1 = 'hello'; """
+ } finally {
+
GetDebugPoint().disableDebugPointForAllBEs("BitmapIndexReader::new_iterator.fail");
+ }
+ } finally {
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
