This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0-var
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0-var by this push:
new 1c5857cec32 [Enhancement](planner) enbale read data opt on origin
planner (#26093)
1c5857cec32 is described below
commit 1c5857cec32ad5637b8a104f3a34df1211a91c62
Author: airborne12 <[email protected]>
AuthorDate: Tue Oct 31 09:38:02 2023 +0800
[Enhancement](planner) enbale read data opt on origin planner (#26093)
---
.../org/apache/doris/planner/OlapScanNode.java | 17 +++++
.../org/apache/doris/planner/OriginalPlanner.java | 75 +++++++++++++++++++---
.../java/org/apache/doris/qe/SessionVariable.java | 6 ++
3 files changed, 88 insertions(+), 10 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index 98490369f03..6c8d35e0588 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -98,6 +98,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -1207,6 +1208,22 @@ public class OlapScanNode extends ScanNode {
output.append(getRuntimeFilterExplainString(false));
}
+ if (!outputColumnUniqueIds.isEmpty()) {
+ output.append(prefix).append("scan columns required by project: ");
+ Iterator<Integer> iterator = outputColumnUniqueIds.iterator();
+ while (iterator.hasNext()) {
+ Integer uniqId = iterator.next();
+ for (SlotDescriptor slotDesc : desc.getSlots()) {
+ if (slotDesc.getColumn() != null &&
slotDesc.getColumn().getUniqueId() == uniqId) {
+ output.append(slotDesc.getColumn().getName());
+ if (iterator.hasNext()) {
+ output.append(", ");
+ }
+ }
+ }
+ }
+ output.append("\n");
+ }
output.append(prefix).append(String.format("partitions=%s/%s,
tablets=%s/%s", selectedPartitionNum,
olapTable.getPartitions().size(), selectedTabletsNum,
totalTabletsNum));
// We print up to 3 tablet, and we print "..." if the number is more
than 3
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
index f77aba25dac..ddfbb876a16 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
@@ -475,6 +475,19 @@ public class OriginalPlanner extends Planner {
}
}
+ private void processExpr(Expr expr, Set<Integer> outputColumnUniqueIds) {
+ if (expr instanceof SlotRef) {
+ SlotRef slotRef = (SlotRef) expr;
+ if (slotRef.getColumn() != null) {
+ outputColumnUniqueIds.add(slotRef.getColumn().getUniqueId());
+ }
+ }
+ ArrayList<Expr> children = expr.getChildren();
+ for (Expr child : children) {
+ processExpr(child, outputColumnUniqueIds); // Recursive call for
each child
+ }
+ }
+
/**
* outputColumnUniqueIds contain columns in OrderByExprs and outputExprs,
* push output column unique id set to olap scan.
@@ -491,19 +504,61 @@ public class OriginalPlanner extends Planner {
// this opt will only work with nereidsPlanner
private void pushOutColumnUniqueIdsToOlapScan(PlanFragment rootFragment,
Analyzer analyzer) {
Set<Integer> outputColumnUniqueIds = new HashSet<>();
- // add '-1' to avoid the optimization incorrect work with
OriginalPlanner,
- // because in the storage layer will skip this optimization if
outputColumnUniqueIds contains '-1',
- // to ensure the optimization only correct work with nereidsPlanner
- outputColumnUniqueIds.add(-1);
+ if
(ConnectContext.get().getSessionVariable().enableIndexDataReadOptOnOrigPlanner)
{
+ ArrayList<Expr> outputExprs = rootFragment.getOutputExprs();
+ for (Expr expr : outputExprs) {
+ processExpr(expr, outputColumnUniqueIds);
+ }
+ for (PlanFragment fragment : fragments) {
+ PlanNode node = fragment.getPlanRoot();
+ PlanNode parent = null;
+ while (node.getChildren().size() != 0) {
+ for (PlanNode childNode : node.getChildren()) {
+ List<SlotId> outputSlotIds =
childNode.getOutputSlotIds();
+ if (outputSlotIds != null) {
+ for (SlotId sid : outputSlotIds) {
+ SlotDescriptor slotDesc =
analyzer.getSlotDesc(sid);
+
outputColumnUniqueIds.add(slotDesc.getUniqueId());
+ }
+ }
+ }
+ // OlapScanNode is the last node.
+ // So, just get the two node and check if they are
SortNode and OlapScan.
+ parent = node;
+ node = node.getChildren().get(0);
+ }
- for (PlanFragment fragment : fragments) {
- PlanNode node = fragment.getPlanRoot();
- if (!(node instanceof OlapScanNode)) {
- continue;
+ if (parent instanceof SortNode) {
+ SortNode sortNode = (SortNode) parent;
+ List<Expr> orderingExprs =
sortNode.getSortInfo().getOrigOrderingExprs();
+ if (orderingExprs != null) {
+ for (Expr expr : orderingExprs) {
+ processExpr(expr, outputColumnUniqueIds);
+ }
+ }
+ }
+ if (!(node instanceof OlapScanNode)) {
+ continue;
+ }
+
+ OlapScanNode scanNode = (OlapScanNode) node;
+ scanNode.setOutputColumnUniqueIds(outputColumnUniqueIds);
}
+ } else {
+ // add '-1' to avoid the optimization incorrect work with
OriginalPlanner,
+ // because in the storage layer will skip this optimization if
outputColumnUniqueIds contains '-1',
+ // to ensure the optimization only correct work with nereidsPlanner
+ outputColumnUniqueIds.add(-1);
+
+ for (PlanFragment fragment : fragments) {
+ PlanNode node = fragment.getPlanRoot();
+ if (!(node instanceof OlapScanNode)) {
+ continue;
+ }
- OlapScanNode scanNode = (OlapScanNode) node;
- scanNode.setOutputColumnUniqueIds(outputColumnUniqueIds);
+ OlapScanNode scanNode = (OlapScanNode) node;
+ scanNode.setOutputColumnUniqueIds(outputColumnUniqueIds);
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 794d379117d..e0e118e308c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -396,6 +396,9 @@ public class SessionVariable implements Serializable,
Writable {
public static final String INVERTED_INDEX_CONJUNCTION_OPT_THRESHOLD =
"inverted_index_conjunction_opt_threshold";
+ public static final String
ENABLE_INDEX_DATA_READ_OPTIMIZED_ON_ORIGINAL_PLANNER
+ = "enable_index_data_read_opt_on_orig_planner";
+
public static final List<String> DEBUG_VARIABLES = ImmutableList.of(
SKIP_DELETE_PREDICATE,
SKIP_DELETE_BITMAP,
@@ -1128,6 +1131,9 @@ public class SessionVariable implements Serializable,
Writable {
+ " use a skiplist to optimize the intersection."})
public int invertedIndexConjunctionOptThreshold = 1000;
+ @VariableMgr.VarAttr(name =
ENABLE_INDEX_DATA_READ_OPTIMIZED_ON_ORIGINAL_PLANNER, needForward = true)
+ public boolean enableIndexDataReadOptOnOrigPlanner = false;
+
@VariableMgr.VarAttr(name = ENABLE_UNIQUE_KEY_PARTIAL_UPDATE, needForward
= true)
public boolean enableUniqueKeyPartialUpdate = false;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
