This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e84272ed43 [improvment](planner) unset common fields to reduce plan
thrift size (#12495)
e84272ed43 is described below
commit e84272ed43256398eb1c071515466f19eaad6821
Author: Mingyu Chen <[email protected]>
AuthorDate: Fri Sep 9 09:02:45 2022 +0800
[improvment](planner) unset common fields to reduce plan thrift size
(#12495)
1. For query with 1656 union, the plan thrift size will be reduced from
400MB+ to 2MB.
This optimization is introduced from #4904, but lost after #9720
2. Disable ExprSubstitutionMap.verify when debug is disable.
So that the plan time of query with 1656 union will be reduced from 20s to
2s
---
.../apache/doris/analysis/ExprSubstitutionMap.java | 20 ++++++++++++--------
.../main/java/org/apache/doris/qe/Coordinator.java | 20 +++++---------------
2 files changed, 17 insertions(+), 23 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java
index fff47d02c2..b1806f3dbf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java
@@ -294,17 +294,21 @@ public final class ExprSubstitutionMap {
* and that all rhs exprs are analyzed.
*/
private void verify() {
- for (int i = 0; i < lhs.size(); ++i) {
- for (int j = i + 1; j < lhs.size(); ++j) {
- if (lhs.get(i).equals(lhs.get(j))) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("verify: smap=" + this.debugString());
+ // This method is very very time consuming, especially when planning
large complex query.
+ // So disable it by default.
+ if (LOG.isDebugEnabled()) {
+ for (int i = 0; i < lhs.size(); ++i) {
+ for (int j = i + 1; j < lhs.size(); ++j) {
+ if (lhs.get(i).equals(lhs.get(j))) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("verify: smap=" + this.debugString());
+ }
+ // TODO(zc): partition by k1, order by k1, there is
failed.
+ // Preconditions.checkState(false);
}
- // TODO(zc): partition by k1, order by k1, there is failed.
- // Preconditions.checkState(false);
}
+ Preconditions.checkState(!checkAnalyzed ||
rhs.get(i).isAnalyzed());
}
- Preconditions.checkState(!checkAnalyzed ||
rhs.get(i).isAnalyzed());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 21af37ccd2..3524256bd4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -155,8 +155,6 @@ public class Coordinator {
// copied from TQueryExecRequest; constant across all fragments
private final TDescriptorTable descTable;
- private final Set<Long> alreadySentBackendIds = Sets.newHashSet();
-
// Why do we use query global?
// When `NOW()` function is in sql, we need only one now(),
// but, we execute `NOW()` distributed.
@@ -396,7 +394,6 @@ public class Coordinator {
}
this.exportFiles.clear();
this.needCheckBackendExecStates.clear();
- this.alreadySentBackendIds.clear();
} finally {
lock.unlock();
}
@@ -748,9 +745,6 @@ public class Coordinator {
} finally {
pair.first.scopedSpan.endSpan();
}
-
- // succeed to send the plan fragment, update the
"alreadySentBackendIds"
- alreadySentBackendIds.add(pair.first.beId);
}
}
@@ -2081,15 +2075,11 @@ public class Coordinator {
* This information can be obtained from the cache of BE.
*/
public void unsetFields() {
- if (alreadySentBackendIds.contains(backend.getId())) {
- this.rpcParams.unsetDescTbl();
- this.rpcParams.unsetCoord();
- this.rpcParams.unsetQueryGlobals();
- this.rpcParams.unsetResourceInfo();
- this.rpcParams.setIsSimplifiedParam(true);
- } else {
- this.rpcParams.setIsSimplifiedParam(false);
- }
+ this.rpcParams.unsetDescTbl();
+ this.rpcParams.unsetCoord();
+ this.rpcParams.unsetQueryGlobals();
+ this.rpcParams.unsetResourceInfo();
+ this.rpcParams.setIsSimplifiedParam(true);
}
// update profile.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
