This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new 7b0313d72e [improvement](storage) For debugging problems: add session
variable skip_storage_engine_merge to treat agg and unique data model as dup
model #12973
7b0313d72e is described below
commit 7b0313d72e896c31f294bd507420c9b29c1b18da
Author: TengJianPing <[email protected]>
AuthorDate: Tue Sep 27 08:32:35 2022 +0800
[improvement](storage) For debugging problems: add session variable
skip_storage_engine_merge to treat agg and unique data model as dup model #12973
For debug purpose:
Add session variable skip_storage_engine_merge, when set to true, tables
of aggregate key model and unique key model will be read as duplicate
key model.
Add session variable skip_delete_predicate, when set to true, rows
deleted with delete statement will be selected.
---
be/src/exec/olap_scanner.cpp | 7 ++++++-
be/src/olap/reader.cpp | 4 ++--
be/src/runtime/runtime_state.h | 9 ++++++++
docs/en/administrator-guide/variables.md | 8 +++++++-
docs/zh-CN/administrator-guide/variables.md | 5 +++++
.../java/org/apache/doris/common/util/Util.java | 4 +++-
.../java/org/apache/doris/qe/SessionVariable.java | 24 ++++++++++++++++++++++
gensrc/thrift/PaloInternalService.thrift | 6 ++++++
8 files changed, 62 insertions(+), 5 deletions(-)
diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp
index a5f29970ea..01f455b46d 100644
--- a/be/src/exec/olap_scanner.cpp
+++ b/be/src/exec/olap_scanner.cpp
@@ -153,7 +153,12 @@ Status OlapScanner::_init_tablet_reader_params(
->rowset_meta()
->is_segments_overlapping());
- _tablet_reader_params.direct_mode = single_version || _aggregation;
+ if (_runtime_state->skip_storage_engine_merge()) {
+ _tablet_reader_params.direct_mode = true;
+ _aggregation = true;
+ } else {
+ _tablet_reader_params.direct_mode = single_version || _aggregation;
+ }
RETURN_IF_ERROR(_init_return_columns(!_tablet_reader_params.direct_mode));
diff --git a/be/src/olap/reader.cpp b/be/src/olap/reader.cpp
index 9b0b7ceec0..c93cbc3971 100644
--- a/be/src/olap/reader.cpp
+++ b/be/src/olap/reader.cpp
@@ -823,8 +823,8 @@ OLAPStatus TabletReader::_init_delete_condition(const
ReaderParams& read_params)
if (read_params.reader_type == READER_CUMULATIVE_COMPACTION) {
return OLAP_SUCCESS;
}
- OLAPStatus ret;
- {
+ OLAPStatus ret = OLAP_SUCCESS;
+ if (read_params.runtime_state &&
!read_params.runtime_state->skip_delete_predicate()) {
ReadLock rdlock(_tablet->get_header_lock());
ret = _delete_handler.init(_tablet->tablet_schema(),
_tablet->delete_predicates(),
read_params.version.second,
this);
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index b9edc56ea0..f96c799898 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -349,6 +349,15 @@ public:
return _query_options.enable_enable_exchange_node_parallel_merge;
}
+ bool skip_storage_engine_merge() const {
+ return _query_options.__isset.skip_storage_engine_merge &&
+ _query_options.skip_storage_engine_merge;
+ }
+
+ bool skip_delete_predicate() const {
+ return _query_options.__isset.skip_delete_predicate &&
_query_options.skip_delete_predicate;
+ }
+
const std::vector<TTabletCommitInfo>& tablet_commit_infos() const {
return _tablet_commit_infos;
}
diff --git a/docs/en/administrator-guide/variables.md
b/docs/en/administrator-guide/variables.md
index 3cd6b1a718..fb106efa2c 100644
--- a/docs/en/administrator-guide/variables.md
+++ b/docs/en/administrator-guide/variables.md
@@ -496,4 +496,10 @@ Translated with www.DeepL.com/Translator (free version)
SM4_128_CFB128,
SM4_128_OFB,
SM4_128_CTR,
-```
\ No newline at end of file
+```
+
+* `skip_storage_engine_merge`
+ For debugging purpose. In vectorized execution engine, in case of problems
of reading data of Aggregate Key model and Unique Key model, setting value to
`true` will read data as Duplicate Key model.
+
+* `skip_delete_predicate`
+ For debugging purpose. In vectorized execution engine, in case of problems
of reading data, setting value to `true` will also read deleted data.
diff --git a/docs/zh-CN/administrator-guide/variables.md
b/docs/zh-CN/administrator-guide/variables.md
index cef0310456..9c56491b15 100644
--- a/docs/zh-CN/administrator-guide/variables.md
+++ b/docs/zh-CN/administrator-guide/variables.md
@@ -490,3 +490,8 @@ SELECT /*+ SET_VAR(query_timeout = 1,
enable_partition_cache=true) */ sleep(3);
用于控制是否进行谓词推导。取值有两种：true 和 false。默认情况下关闭，系统不在进行谓词推导，采用原始的谓词进行相关操作。设置为 true
后，进行谓词扩展。
+* `skip_storage_engine_merge`
+ 用于调试目的。在向量化执行引擎中，当发现读取Aggregate Key模型或者Unique
Key模型的数据结果有问题的时候，把此变量的值设置为`true`，将会把Aggregate Key模型或者Unique Key模型的数据当成Duplicate
Key模型读取。
+
+* `skip_delete_predicate`
+ 用于调试目的。在向量化执行引擎中，当发现读取表的数据结果有误的时候，把此变量的值设置为`true`，将会把被删除的数据当成正常数据读取。
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
index 1c5daa9770..c5de3e9123 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
@@ -446,7 +446,9 @@ public class Util {
}
public static boolean showHiddenColumns() {
- return ConnectContext.get() != null &&
ConnectContext.get().getSessionVariable().showHiddenColumns();
+ return ConnectContext.get() != null && (
+ ConnectContext.get().getSessionVariable().showHiddenColumns()
+ ||
ConnectContext.get().getSessionVariable().skipStorageEngineMerge());
}
public static String escapeSingleRegex(String s) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 6af37a639b..089047730f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -191,6 +191,10 @@ public class SessionVariable implements Serializable,
Writable {
public static final String NUM_FREE_BLOCK_IN_SCAN =
"num_free_block_in_scan";
+ public static final String SKIP_STORAGE_ENGINE_MERGE =
"skip_storage_engine_merge";
+
+ public static final String SKIP_DELETE_PREDICATE = "skip_delete_predicate";
+
// session origin value
public Map<Field, String> sessionOriginValue = new HashMap<Field,
String>();
// check stmt is or not [select /*+ SET_VAR(...)*/ ...]
@@ -468,6 +472,18 @@ public class SessionVariable implements Serializable,
Writable {
@VariableMgr.VarAttr(name = NUM_FREE_BLOCK_IN_SCAN)
public int numFreeBlockInScan = 12;
+ /**
+ * For debugg purpose, dont' merge unique key and agg key when reading
data.
+ */
+ @VariableMgr.VarAttr(name = SKIP_STORAGE_ENGINE_MERGE)
+ public boolean skipStorageEngineMerge = false;
+
+ /**
+ * For debugg purpose, skip delte predicate when reading data.
+ */
+ @VariableMgr.VarAttr(name = SKIP_DELETE_PREDICATE)
+ public boolean skipDeletePredicate = false;
+
public String getBlockEncryptionMode() {
return blockEncryptionMode;
@@ -768,6 +784,10 @@ public class SessionVariable implements Serializable,
Writable {
this.showHiddenColumns = showHiddenColumns;
}
+ public boolean skipStorageEngineMerge() {
+ return skipStorageEngineMerge;
+ }
+
public boolean isAllowPartitionColumnNullable() {
return allowPartitionColumnNullable;
}
@@ -977,6 +997,10 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setResourceLimit(resourceLimit);
}
+ tResult.setSkipStorageEngineMerge(skipStorageEngineMerge);
+
+ tResult.setSkipDeletePredicate(skipDeletePredicate);
+
return tResult;
}
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index 689cf07919..900b904ea0 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -165,6 +165,12 @@ struct TQueryOptions {
44: optional bool trim_tailing_spaces_for_external_table_query = false
47: optional i32 num_free_block_in_scan
+
+ // For debug purpose, dont' merge unique key and agg key when reading data.
+ 48: optional bool skip_storage_engine_merge = false
+
+ // For debug purpose, skip delete predicates when reading data
+ 49: optional bool skip_delete_predicate = false
}
// A scan range plus the parameters needed to execute that scan.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
