This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 815349d3502 [Feature](inverted index) add no need read data optimize
config (#36686)
815349d3502 is described below
commit 815349d350284aee15f87257a305e8af0ad686a3
Author: airborne12 <[email protected]>
AuthorDate: Sat Jun 22 11:41:06 2024 +0800
[Feature](inverted index) add no need read data optimize config (#36686)
## Proposed changes
Add an FE session variable for the no-read-data optimization in
segment_iterator to prevent incorrect optimizations.
---
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 6 ++++++
.../main/java/org/apache/doris/qe/SessionVariable.java | 15 +++++++++++++++
gensrc/thrift/PaloInternalService.thrift | 2 ++
3 files changed, 23 insertions(+)
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 47d5fe964d0..a8a9758afcd 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -1198,6 +1198,9 @@ Status
SegmentIterator::_apply_inverted_index_on_block_column_predicate(
}
bool SegmentIterator::_need_read_data(ColumnId cid) {
+ if (_opts.runtime_state &&
!_opts.runtime_state->query_options().enable_no_need_read_data_opt) {
+ return true;
+ }
// only support DUP_KEYS and UNIQUE_KEYS with MOW
if (!((_opts.tablet_schema->keys_type() == KeysType::DUP_KEYS ||
(_opts.tablet_schema->keys_type() == KeysType::UNIQUE_KEYS &&
@@ -2830,6 +2833,9 @@ void
SegmentIterator::_calculate_pred_in_remaining_conjunct_root(
bool SegmentIterator::_no_need_read_key_data(ColumnId cid,
vectorized::MutableColumnPtr& column,
size_t nrows_read) {
+ if (_opts.runtime_state &&
!_opts.runtime_state->query_options().enable_no_need_read_data_opt) {
+ return false;
+ }
if (_opts.tablet_schema->keys_type() != KeysType::DUP_KEYS) {
return false;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index a72ad4f9dc2..37d039f8aeb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -404,6 +404,7 @@ public class SessionVariable implements Serializable,
Writable {
= "enable_common_expr_pushdown_for_inverted_index";
public static final String ENABLE_PUSHDOWN_COUNT_ON_INDEX =
"enable_count_on_index_pushdown";
+ public static final String ENABLE_NO_NEED_READ_DATA_OPT =
"enable_no_need_read_data_opt";
public static final String GROUP_BY_AND_HAVING_USE_ALIAS_FIRST =
"group_by_and_having_use_alias_first";
public static final String DROP_TABLE_IF_CTAS_FAILED =
"drop_table_if_ctas_failed";
@@ -1441,6 +1442,11 @@ public class SessionVariable implements Serializable,
Writable {
"是否启用count_on_index pushdown。", "Set whether to pushdown
count_on_index."})
public boolean enablePushDownCountOnIndex = true;
+ // Whether enable no need read data opt in segment_iterator.
+ @VariableMgr.VarAttr(name = ENABLE_NO_NEED_READ_DATA_OPT, needForward =
true, description = {
+ "是否启用no_need_read_data opt。", "Set whether to enable
no_need_read_data opt."})
+ public boolean enableNoNeedReadDataOpt = true;
+
// Whether enable pushdown minmax to scan node of unique table.
@VariableMgr.VarAttr(name = ENABLE_PUSHDOWN_MINMAX_ON_UNIQUE, needForward
= true, description = {
"是否启用pushdown minmax on unique table。", "Set whether to pushdown
minmax on unique table."})
@@ -3248,6 +3254,14 @@ public class SessionVariable implements Serializable,
Writable {
this.enablePushDownCountOnIndex = enablePushDownCountOnIndex;
}
+ public boolean isEnableNoNeedReadDataOpt() {
+ return enableNoNeedReadDataOpt;
+ }
+
+ public void setEnableNoNeedReadDataOpt(boolean enableNoNeedReadDataOpt) {
+ this.enableNoNeedReadDataOpt = enableNoNeedReadDataOpt;
+ }
+
public int getMaxTableCountUseCascadesJoinReorder() {
return this.maxTableCountUseCascadesJoinReorder;
}
@@ -3383,6 +3397,7 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setEnableInvertedIndexQuery(enableInvertedIndexQuery);
tResult.setEnableCommonExprPushdownForInvertedIndex(enableCommonExpPushDownForInvertedIndex);
+ tResult.setEnableNoNeedReadDataOpt(enableNoNeedReadDataOpt);
if (dryRunQuery) {
tResult.setDryRunQuery(true);
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index 9df69dd995f..441f695e18e 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -305,6 +305,8 @@ struct TQueryOptions {
114: optional bool enable_parallel_result_sink = false;
115: optional bool enable_short_circuit_query_access_column_store = false;
+
+ 116: optional bool enable_no_need_read_data_opt = true;
// For cloud, to control if the content would be written into file cache
1000: optional bool disable_file_cache = false
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
