This is an automated email from the ASF dual-hosted git repository.
zhangchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 55519965281 [fix](inverted index) skip read index column data only for
DUP and MOW table (#32594)
55519965281 is described below
commit 555199652818905991fe401e5f10a106d3a0a78d
Author: Kang <[email protected]>
AuthorDate: Thu Mar 21 17:44:56 2024 +0800
[fix](inverted index) skip read index column data only for DUP and MOW
table (#32594)
---
be/src/olap/iterators.h | 1 +
be/src/olap/rowset/beta_rowset_reader.cpp | 2 +
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 6 +
.../test_index_skip_read_data.out | 83 +++++++++++++
.../test_index_skip_read_data.groovy | 131 +++++++++++++++++++++
5 files changed, 223 insertions(+)
diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h
index a7fb43cff58..e4b62d157d1 100644
--- a/be/src/olap/iterators.h
+++ b/be/src/olap/iterators.h
@@ -97,6 +97,7 @@ public:
int block_row_max = 4096 - 32; // see
https://github.com/apache/doris/pull/11816
TabletSchemaSPtr tablet_schema = nullptr;
+ bool enable_unique_key_merge_on_write = false;
bool record_rowids = false;
// flag for enable topn opt
bool use_topn_opt = false;
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp
b/be/src/olap/rowset/beta_rowset_reader.cpp
index 8cd70689d7b..a0dff2613ae 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -215,6 +215,8 @@ Status
BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
}
_read_options.use_page_cache = _read_context->use_page_cache;
_read_options.tablet_schema = _read_context->tablet_schema;
+ _read_options.enable_unique_key_merge_on_write =
+ _read_context->enable_unique_key_merge_on_write;
_read_options.record_rowids = _read_context->record_rowids;
_read_options.use_topn_opt = _read_context->use_topn_opt;
_read_options.topn_filter_source_node_ids =
_read_context->topn_filter_source_node_ids;
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index ad5b25f6620..fd503fbcd5a 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -1086,6 +1086,12 @@ Status
SegmentIterator::_apply_inverted_index_on_block_column_predicate(
}
bool SegmentIterator::_need_read_data(ColumnId cid) {
+ // only support DUP_KEYS and UNIQUE_KEYS with MOW
+ if (!((_opts.tablet_schema->keys_type() == KeysType::DUP_KEYS ||
+ (_opts.tablet_schema->keys_type() == KeysType::UNIQUE_KEYS &&
+ _opts.enable_unique_key_merge_on_write)))) {
+ return true;
+ }
// if there is delete predicate, we always need to read data
if (_opts.delete_condition_predicates->num_of_column_predicate() > 0) {
return true;
diff --git
a/regression-test/data/inverted_index_p0/test_index_skip_read_data.out
b/regression-test/data/inverted_index_p0/test_index_skip_read_data.out
new file mode 100644
index 00000000000..bb79d437b2a
--- /dev/null
+++ b/regression-test/data/inverted_index_p0/test_index_skip_read_data.out
@@ -0,0 +1,83 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql10 --
+1 20 300
+1 20 400
+
+-- !sql11 --
+1 20 300
+1 20 400
+
+-- !sql12 --
+1 20 300
+1 20 400
+
+-- !sql13 --
+1 20 300
+
+-- !sql14 --
+1 20 400
+
+-- !sql15 --
+20
+20
+
+-- !sql16 --
+1
+1
+
+-- !sql17 --
+1 20
+
+-- !sql18 --
+1 20
+
+-- !sql20 --
+1 20 400
+
+-- !sql21 --
+1 20 400
+
+-- !sql22 --
+1 20 400
+
+-- !sql23 --
+
+-- !sql24 --
+1 20 400
+
+-- !sql25 --
+20
+
+-- !sql26 --
+1
+
+-- !sql27 --
+
+-- !sql28 --
+1 20
+
+-- !sql30 --
+1 20 400
+
+-- !sql31 --
+1 20 400
+
+-- !sql32 --
+1 20 400
+
+-- !sql33 --
+
+-- !sql34 --
+1 20 400
+
+-- !sql35 --
+20
+
+-- !sql36 --
+1
+
+-- !sql37 --
+
+-- !sql38 --
+1 20
+
diff --git
a/regression-test/suites/inverted_index_p0/test_index_skip_read_data.groovy
b/regression-test/suites/inverted_index_p0/test_index_skip_read_data.groovy
new file mode 100644
index 00000000000..70213910934
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/test_index_skip_read_data.groovy
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_index_skip_read_data", "p0"){
+ def indexTbName1 = "test_index_skip_read_data_dup"
+ def indexTbName2 = "test_index_skip_read_data_mow"
+ def indexTbName3 = "test_index_skip_read_data_mor"
+
+
+ // dup
+ sql "DROP TABLE IF EXISTS ${indexTbName1}"
+
+ sql """
+ CREATE TABLE ${indexTbName1} (
+ `k1` int(11) NULL COMMENT "",
+ `k2` varchar(20) NULL COMMENT "",
+ `data` text NULL COMMENT "",
+ INDEX idx_k1 (`k1`) USING INVERTED COMMENT '',
+ INDEX idx_k2 (`k2`) USING BITMAP COMMENT '',
+ INDEX idx_data (`data`) USING INVERTED COMMENT ''
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`k1`, `k2`)
+ COMMENT "OLAP"
+ DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "disable_auto_compaction" = "true"
+ );
+ """
+
+ sql """ INSERT INTO ${indexTbName1} VALUES (1, 20, 300); """
+ sql """ INSERT INTO ${indexTbName1} VALUES (1, 20, 400); """
+
+ qt_sql10 """ SELECT * FROM ${indexTbName1} ORDER BY k1,k2,data; """
+ qt_sql11 """ SELECT * FROM ${indexTbName1} WHERE k1 = 1 ORDER BY
k1,k2,data; """
+ qt_sql12 """ SELECT * FROM ${indexTbName1} WHERE k2 = 20 ORDER BY
k1,k2,data; """
+ qt_sql13 """ SELECT * FROM ${indexTbName1} WHERE data = 300 ORDER BY
k1,k2,data; """
+ qt_sql14 """ SELECT * FROM ${indexTbName1} WHERE data = 400 ORDER BY
k1,k2,data; """
+ qt_sql15 """ SELECT k2 FROM ${indexTbName1} WHERE k1 = 1 ORDER BY
k1,k2,data; """
+ qt_sql16 """ SELECT k1 FROM ${indexTbName1} WHERE k2 = 20 ORDER BY
k1,k2,data; """
+ qt_sql17 """ SELECT k1, k2 FROM ${indexTbName1} WHERE data = 300 ORDER BY
k1,k2,data; """
+ qt_sql18 """ SELECT k1, k2 FROM ${indexTbName1} WHERE data = 400 ORDER BY
k1,k2,data; """
+
+
+
+ // mow
+ sql "DROP TABLE IF EXISTS ${indexTbName2}"
+
+ sql """
+ CREATE TABLE ${indexTbName2} (
+ `k1` int(11) NULL COMMENT "",
+ `k2` varchar(20) NULL COMMENT "",
+ `data` text NULL COMMENT "",
+ INDEX idx_k1 (`k1`) USING INVERTED COMMENT '',
+ INDEX idx_k2 (`k2`) USING BITMAP COMMENT '',
+ INDEX idx_data (`data`) USING INVERTED COMMENT ''
+ ) ENGINE=OLAP
+ UNIQUE KEY(`k1`, `k2`)
+ COMMENT "OLAP"
+ DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "enable_unique_key_merge_on_write" = "true",
+ "disable_auto_compaction" = "true"
+ );
+ """
+
+ sql """ INSERT INTO ${indexTbName2} VALUES (1, 20, 300); """
+ sql """ INSERT INTO ${indexTbName2} VALUES (1, 20, 400); """
+
+ qt_sql20 """ SELECT * FROM ${indexTbName2} ORDER BY k1,k2,data; """
+ qt_sql21 """ SELECT * FROM ${indexTbName2} WHERE k1 = 1 ORDER BY
k1,k2,data; """
+ qt_sql22 """ SELECT * FROM ${indexTbName2} WHERE k2 = 20 ORDER BY
k1,k2,data; """
+ qt_sql23 """ SELECT * FROM ${indexTbName2} WHERE data = 300 ORDER BY
k1,k2,data; """
+ qt_sql24 """ SELECT * FROM ${indexTbName2} WHERE data = 400 ORDER BY
k1,k2,data; """
+ qt_sql25 """ SELECT k2 FROM ${indexTbName2} WHERE k1 = 1 ORDER BY
k1,k2,data; """
+ qt_sql26 """ SELECT k1 FROM ${indexTbName2} WHERE k2 = 20 ORDER BY
k1,k2,data; """
+ qt_sql27 """ SELECT k1, k2 FROM ${indexTbName2} WHERE data = 300 ORDER BY
k1,k2,data; """
+ qt_sql28 """ SELECT k1, k2 FROM ${indexTbName2} WHERE data = 400 ORDER BY
k1,k2,data; """
+
+
+ // mor
+ sql "DROP TABLE IF EXISTS ${indexTbName3}"
+
+ sql """
+ CREATE TABLE ${indexTbName3} (
+ `k1` int(11) NULL COMMENT "",
+ `k2` varchar(20) NULL COMMENT "",
+ `data` text NULL COMMENT "",
+ INDEX idx_k1 (`k1`) USING INVERTED COMMENT '',
+ INDEX idx_k2 (`k2`) USING BITMAP COMMENT '',
+ INDEX idx_data (`data`) USING INVERTED COMMENT ''
+ ) ENGINE=OLAP
+ UNIQUE KEY(`k1`, `k2`)
+ COMMENT "OLAP"
+ DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "enable_unique_key_merge_on_write" = "false",
+ "disable_auto_compaction" = "true"
+ );
+ """
+
+ sql """ INSERT INTO ${indexTbName3} VALUES (1, 20, 300); """
+ sql """ INSERT INTO ${indexTbName3} VALUES (1, 20, 400); """
+
+ qt_sql30 """ SELECT * FROM ${indexTbName3} ORDER BY k1,k2,data; """
+ qt_sql31 """ SELECT * FROM ${indexTbName3} WHERE k1 = 1 ORDER BY
k1,k2,data; """
+ qt_sql32 """ SELECT * FROM ${indexTbName3} WHERE k2 = 20 ORDER BY
k1,k2,data; """
+ qt_sql33 """ SELECT * FROM ${indexTbName3} WHERE data = 300 ORDER BY
k1,k2,data; """
+ qt_sql34 """ SELECT * FROM ${indexTbName3} WHERE data = 400 ORDER BY
k1,k2,data; """
+ qt_sql35 """ SELECT k2 FROM ${indexTbName3} WHERE k1 = 1 ORDER BY
k1,k2,data; """
+ qt_sql36 """ SELECT k1 FROM ${indexTbName3} WHERE k2 = 20 ORDER BY
k1,k2,data; """
+ qt_sql37 """ SELECT k1, k2 FROM ${indexTbName3} WHERE data = 300 ORDER BY
k1,k2,data; """
+ qt_sql38 """ SELECT k1, k2 FROM ${indexTbName3} WHERE data = 400 ORDER BY
k1,k2,data; """
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
