This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6bf5fc6db5 [improvement](storage) For debugging problems: add session
variable `skip_storage_engine_merge` to treat agg and unique data model as dup
model (#11952)
6bf5fc6db5 is described below
commit 6bf5fc6db50f1c543b9c5b50bef00ceb4b60ff3e
Author: TengJianPing <[email protected]>
AuthorDate: Tue Sep 13 19:18:56 2022 +0800
[improvement](storage) For debugging problems: add session variable
`skip_storage_engine_merge` to treat agg and unique data model as dup model
(#11952)
For debug purpose:
Add session variable skip_storage_engine_merge, when set to true, tables of
aggregate key model and unique key model will be read as duplicate key model.
Add session variable skip_delete_predicate, when set to true, rows deleted
with delete statement will be selected.
---
be/src/runtime/runtime_state.h | 9 ++++++++
be/src/vec/exec/volap_scanner.cpp | 16 ++++++++++-----
docs/en/docs/advanced/variables.md | 6 ++++++
docs/zh-CN/docs/advanced/variables.md | 6 ++++++
.../java/org/apache/doris/common/util/Util.java | 4 +++-
.../java/org/apache/doris/qe/SessionVariable.java | 24 ++++++++++++++++++++++
gensrc/thrift/PaloInternalService.thrift | 6 ++++++
7 files changed, 65 insertions(+), 6 deletions(-)
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index 8b49f0a161..1b7042815a 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -349,6 +349,15 @@ public:
return segment_v2::CompressionTypePB::SNAPPY;
}
+ bool skip_storage_engine_merge() const {
+ return _query_options.__isset.skip_storage_engine_merge &&
+ _query_options.skip_storage_engine_merge;
+ }
+
+ bool skip_delete_predicate() const {
+ return _query_options.__isset.skip_delete_predicate &&
_query_options.skip_delete_predicate;
+ }
+
const std::vector<TTabletCommitInfo>& tablet_commit_infos() const {
return _tablet_commit_infos;
}
diff --git a/be/src/vec/exec/volap_scanner.cpp
b/be/src/vec/exec/volap_scanner.cpp
index efa80b36ac..4b820aa1ce 100644
--- a/be/src/vec/exec/volap_scanner.cpp
+++ b/be/src/vec/exec/volap_scanner.cpp
@@ -171,7 +171,11 @@ Status VOlapScanner::_init_tablet_reader_params(
->rowset_meta()
->is_segments_overlapping());
- _tablet_reader_params.direct_mode = _aggregation || single_version;
+ if (_runtime_state->skip_storage_engine_merge()) {
+ _tablet_reader_params.direct_mode = true;
+ } else {
+ _tablet_reader_params.direct_mode = _aggregation || single_version;
+ }
RETURN_IF_ERROR(_init_return_columns(!_tablet_reader_params.direct_mode));
@@ -192,10 +196,12 @@ Status VOlapScanner::_init_tablet_reader_params(
std::copy(function_filters.cbegin(), function_filters.cend(),
std::inserter(_tablet_reader_params.function_filters,
_tablet_reader_params.function_filters.begin()));
- auto& delete_preds = _tablet->delete_predicates();
- std::copy(delete_preds.cbegin(), delete_preds.cend(),
- std::inserter(_tablet_reader_params.delete_predicates,
- _tablet_reader_params.delete_predicates.begin()));
+ if (!_runtime_state->skip_delete_predicate()) {
+ auto& delete_preds = _tablet->delete_predicates();
+ std::copy(delete_preds.cbegin(), delete_preds.cend(),
+ std::inserter(_tablet_reader_params.delete_predicates,
+
_tablet_reader_params.delete_predicates.begin()));
+ }
// Merge the columns in delete predicate that not in latest schema in to
current tablet schema
for (auto& del_pred_rs : _tablet_reader_params.delete_predicates) {
diff --git a/docs/en/docs/advanced/variables.md
b/docs/en/docs/advanced/variables.md
index 5b388c8a05..537a271e73 100644
--- a/docs/en/docs/advanced/variables.md
+++ b/docs/en/docs/advanced/variables.md
@@ -526,3 +526,9 @@ Translated with www.DeepL.com/Translator (free version)
* `trim_tailing_spaces_for_external_table_query`
Used to control whether trim the tailing spaces while quering Hive external
tables. The default is false.
+
+* `skip_storage_engine_merge`
+ For debugging purpose. In vectorized execution engine, in case of problems
of reading data of Aggregate Key model and Unique Key model, setting value to
`true` will read data as Duplicate Key model.
+
+* `skip_delete_predicate`
+ For debugging purpose. In vectorized execution engine, in case of problems
of reading data, setting value to `true` will also read deleted data.
diff --git a/docs/zh-CN/docs/advanced/variables.md
b/docs/zh-CN/docs/advanced/variables.md
index c548f452f9..6ce5eb836e 100644
--- a/docs/zh-CN/docs/advanced/variables.md
+++ b/docs/zh-CN/docs/advanced/variables.md
@@ -515,3 +515,9 @@ SELECT /*+ SET_VAR(query_timeout = 1,
enable_partition_cache=true) */ sleep(3);
- `trim_tailing_spaces_for_external_table_query`
用于控制查询Hive外表时是否过滤掉字段末尾的空格。默认为false。
+
+* `skip_storage_engine_merge`
+ 用于调试目的。在向量化执行引擎中，当发现读取Aggregate Key模型或者Unique
Key模型的数据结果有问题的时候，把此变量的值设置为`true`，将会把Aggregate Key模型或者Unique Key模型的数据当成Duplicate
Key模型读取。
+
+* `skip_delete_predicate`
+ 用于调试目的。在向量化执行引擎中，当发现读取表的数据结果有误的时候，把此变量的值设置为`true`，将会把被删除的数据当成正常数据读取。
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
index 0d5cf4eed7..f1f6fc6282 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
@@ -455,7 +455,9 @@ public class Util {
}
public static boolean showHiddenColumns() {
- return ConnectContext.get() != null &&
ConnectContext.get().getSessionVariable().showHiddenColumns();
+ return ConnectContext.get() != null && (
+ ConnectContext.get().getSessionVariable().showHiddenColumns()
+ ||
ConnectContext.get().getSessionVariable().skipStorageEngineMerge());
}
public static String escapeSingleRegex(String s) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index da79ae0410..a12ee9059c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -211,6 +211,10 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_LOCAL_EXCHANGE = "enable_local_exchange";
+ public static final String SKIP_STORAGE_ENGINE_MERGE =
"skip_storage_engine_merge";
+
+ public static final String SKIP_DELETE_PREDICATE = "skip_delete_predicate";
+
// session origin value
public Map<Field, String> sessionOriginValue = new HashMap<Field,
String>();
// check stmt is or not [select /*+ SET_VAR(...)*/ ...]
@@ -530,6 +534,18 @@ public class SessionVariable implements Serializable,
Writable {
@VariableMgr.VarAttr(name = ENABLE_LOCAL_EXCHANGE)
public boolean enableLocalExchange = false;
+ /**
+ * For debugg purpose, dont' merge unique key and agg key when reading
data.
+ */
+ @VariableMgr.VarAttr(name = SKIP_STORAGE_ENGINE_MERGE)
+ public boolean skipStorageEngineMerge = false;
+
+ /**
+ * For debugg purpose, skip delte predicate when reading data.
+ */
+ @VariableMgr.VarAttr(name = SKIP_DELETE_PREDICATE)
+ public boolean skipDeletePredicate = false;
+
public String getBlockEncryptionMode() {
return blockEncryptionMode;
}
@@ -843,6 +859,10 @@ public class SessionVariable implements Serializable,
Writable {
this.showHiddenColumns = showHiddenColumns;
}
+ public boolean skipStorageEngineMerge() {
+ return skipStorageEngineMerge;
+ }
+
public boolean isAllowPartitionColumnNullable() {
return allowPartitionColumnNullable;
}
@@ -1124,6 +1144,10 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setFragmentTransmissionCompressionCodec(fragmentTransmissionCompressionCodec);
tResult.setEnableLocalExchange(enableLocalExchange);
+ tResult.setSkipStorageEngineMerge(skipStorageEngineMerge);
+
+ tResult.setSkipDeletePredicate(skipDeletePredicate);
+
return tResult;
}
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index c1c9f97d07..d9911e6141 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -169,6 +169,12 @@ struct TQueryOptions {
46: optional string fragment_transmission_compression_codec;
47: optional bool enable_local_exchange;
+
+ // For debug purpose, dont' merge unique key and agg key when reading data.
+ 48: optional bool skip_storage_engine_merge = false
+
+ // For debug purpose, skip delete predicates when reading data
+ 49: optional bool skip_delete_predicate = false
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
